@letsscrapedata/utils/dist/index.d.ts

This package is mainly used by LetsScrapeData App

declare function getCurrentUnixTime(): number;
declare function getDaysOfYear(offset?: number): 366 | 365;
declare function getDaysOfMonth(offset?: number): number;
/**
 * @description return "yyyy-MM-dd HH:mm:ss" of date
 * @return {string}
 * @param {Date} date default new Date()
 */
declare function getLocalDateString(date?: Date): string;
/**
 * @description 返回某时间或当前时间UTC的"yyyy-MM-dd HH:mm:ss"格式字符串
 * @return {string}
 * @param {Date} date default new Date()
 */
declare function getUTCDateString(date?: Date): string;
/**
 * return "yyyyMMddHHmmss"
 * @param date
 * @returns
 */
declare function getLocalDateNumber(date?: Date): number;
declare function getMonthSecondsOfUnixTime(unixTime: number): number;
declare function getUnixTimeOfDay(offset?: number): number;
declare function getUnixTimeOfWeek(offset?: number): number;
declare function getUnixTimeOfMonth(offset?: number): number;
declare function getUnixTimeOfYear(offset?: number): number;
declare function getUtcUnixTimeOfDay(offset?: number): number;
declare function getUtcUnixTimeOfWeek(offset?: number): number;
declare function getUtcUnixTimeOfMonth(offset?: number): number;
declare function getUtcUnixTimeOfYear(offset?: number): number;
declare function getYyyyMmDdDateNum(date?: Date, diffDays?: number): number;
declare function addTime(initialTime: Date, addSeconds: number): Date;
/**
 * @param time1
 * @param time2
 * @returns
 * * 1: time1 > time2
 * * 0: time1 === time2
 * * -1: time1 < time2
 */
declare function compareTime(time1: Date, time2: Date): 0 | 1 | -1;

declare function fileExist(path: string): Promise<boolean>;
type FileType = "general" | "directory" | "block" | "character" | "fifo" | "socket" | "link" | "any";
declare function filesInDir(path: string, fileType?: FileType, prefix?: string, extname?: string): Promise<string[]>;
declare function isFile(path: string): Promise<boolean>;
declare function isDirectory(path: string): Promise<boolean>;
declare function mkdir(path: string): Promise<boolean>;
/**
 * @description 检查目录是否存在、具有相应权限，如需要自动创建目录
 * @return {Promise<boolean>}
 * @param {string} dirName
 * @param {boolean} writeFlag default false, 是否具有读写权限
 * @param {boolean} createFlag default false, 如果目录不存在是否自动创建
 * @param {boolean} recursiveFlag default false, 创建目录时，是否逐级创建，false表示只能创建最后一级目录
 */
declare function fsCheckOrCreateDir(dirName: string, writeFlag?: boolean, createFlag?: boolean, recursiveFlag?: boolean): Promise<boolean>;
declare function fsCheckOrCreateSubdir(baseDir: string, subDir: string, sep?: string): Promise<boolean>;
declare function removeFile(filename: string): Promise<boolean>;
declare function isDirectorySync(path: string): boolean;
declare function isFileSync(path: string): boolean;
declare function existsSync(path: string): boolean;

/**
 * suggested level: DBG/INF/WRN/ERR
 */
declare enum LogLevel {
    DBG = 0,
    INF = 1,
    WRN = 2,
    ERR = 3
}
/**
 * No need to judge whether to write logs, refer to consoleLogFun
 */
type LogFunction = (level: LogLevel, ...args: any[]) => Promise<boolean>;
type LevelLogFunction = (...args: any[]) => Promise<boolean>;
declare function setLogLevel(newLevel: LogLevel): boolean;
/**
 * logFunction with name will be added or replaced
 * @param name
 * @param logFunction
 * @returns
 */
declare function addLogFunction(name: string, logFunction: LogFunction): boolean;
declare function getNamesOfLogFunctions(): string[];
/**
 * names of default logFunctions:
 * * "console":
 * * "defaultFile": if log directory exists and tryToAddDefaultFileLogFun is called
 * @param name
 * @returns
 */
declare function removeLogFunction(name: string): boolean;
declare const log: LogFunction;
interface GeneralFileLogOptions {
    /**
     * directory of log file, which must exist!!!
     * @defualt "log"
     */
    logFilenameDir?: string;
    /**
     * @default "lsd-"
     */
    logFilenamePrefix?: string;
    /**
     * yyyy: year such as "2024"
     * MM: month such as "01" or "12"
     * dd: day, such as "01" or "31"
     * HH: hour, such as "01" or "23"
     * mm: minute, such as "01" or "59"
     * ss: second, such as "01" or "59"
     * @default "yyyyMMdd-HHmmss"
     */
    dateFormat?: string;
    /**
     * max size (bytes) of log file, 0 - unlimited
     * @default 10240000, 10Mb
     */
    maxSize?: number;
    /**
     * intervals (seconds)
     * valid when maxSize is greater 0 and intervals > 60
     * @default 600
     */
    intervals?: number;
    /**
     * whether the time is includes in message
     * @default true
     */
    datetimeIncluded?: boolean;
}
declare class GeneralFileLog {
    #private;
    constructor(options?: GeneralFileLogOptions);
    /**
     * get date string according to dateFormat
     * @param dateFormat
     * * yyyy: year such as "2024"
     * * MM: month such as "01" or "12"
     * * dd: day, such as "01" or "31"
     * * HH: hour, such as "01" or "23"
     * * mm: minute, such as "01" or "59"
     * * ss: second, such as "01" or "59"
     * @param date default new Date()
     */
    static getDateString(dateFormat: string, date?: Date): string;
    static getMessage(level: LogLevel, args: any[], datetimeIncluded?: boolean): string;
    /**
     * add a new log file
     * * the directory must exist, or return false!
     * * if the file exists, it must be a general text file and you have write privilege
     * * if the file does not exist, it will be created
     * @param path
     */
    log(level: LogLevel, ...args: any[]): Promise<boolean>;
}
/**
 * if log directory exists, add default file logFunction
 */
declare function tryToAddDefaultFileLogFun(): boolean;

declare function getRandomInt(minNum: number, maxNum: number): number;
declare function getHexStrOfIntNum(intNum: number, strNum?: number): string;
declare function getComplementOfTwoArray(array1: any[], array2: any[]): any[];
declare function getDifferenceOfTwoArray(array1: any[], array2: any[]): any[];
declare function getIntersectionOfTwoArray(array1: any[], array2: any[]): any[];
declare function getUnionOfTwoArray(array1: any[], array2: any[]): any[];

declare function sleep(miliseconds: number): Promise<void>;
declare function unreachable(x: never): never;
declare function getRandomVcCode(vcCodeLength?: number, strScopeLen?: number): string;
declare function getMemberOfObject(origObj: any, key1: string, key2?: string, key3?: string, key4?: string, key5?: string, key6?: string): any;
/**
 * @description return JSON.parse(str), or return defaulVal if str is empty or there is error
 * @return {any}
 * @param {string} str
 * @param {any} defaultVal default {}
 */
declare function getJsonParseOfStr(str: string, defaultVal?: any): any;
/**
 * @description 返回一个长度为length数组的随机下标数组
 * @return {[number]}
 * @param {number} length length of array, that should be greater than 0
 */
declare function getRandomArrayIdx(length: number): number[];
/**
 * @description return the unique array of original array
 * @return {[]}
 */
declare function getUniqueArray(origArray: any[]): any[];
declare function convertMapToObj(map: Map<string, any>): any;

declare function checkDns(domainName: string): Promise<boolean>;
declare function checkPing(host: string): Promise<boolean>;
declare function getDomainName(url: string): Promise<string>;

declare function decodeFromBase64(base64Str: string): any;
declare function encodeToBase64(origData: any): string;
declare function zipOrigStrToZippedBase64(origStr: string): Promise<string>;
declare function unzipZippedBase64ToOrigStr(zippedBase64: string): Promise<string>;
declare function getHashCodeOfString(str: string, divisor?: number): number;

declare function execCommand(command: string): Promise<string>;
/**
 * get pids of the processes listening on TCP/UDP port
 * @param port
 * @param protocol default "TCP"
 * @returns
 */
declare function getPidsListeningOnPort(port: number, protocol?: "TCP" | "UDP"): Promise<number[]>;
/**
 * get the total cpu and memory of pid and its decendants
 * @param pid
 * @param memoryUnit default "bytes"
 * @returns return {cpu: 0, memory: 0} if pid does not exist
 */
declare function getPerformanceOfPidTree(pid: number, memoryUnit?: "bytes" | "MB" | "GB"): Promise<{
    cpu: number;
    memory: number;
}>;

type PathType = "basename" | "hashcode" | "hostname_hashcode" | "hostnameHashcode" | "hostname_path_name" | "hostnamePath_name" | "hostnamePathname" | "hostname_title" | "hostnameTitle" | "path_name" | "pathname" | "title";
type UrlAttrName = "href" | "origin" | "protocol" | "username" | "password" | "host" | "pathname" | "search" | "hash" | "hostname" | "port";
type HashMethod = "MD4" | "MD5" | "SHA1" | "SHA224" | "SHA256" | "SHA384" | "SHA512" | "uuidv1" | "uuidv4";
declare class LsdUrl {
    #private;
    /**
     * Get absolute url of oriUrl
     ** return origUrl if origUrl starts with "http"
     * @param pageUrl
     * @param origUrl
     * @returns  return "" if invalid paras or failed
     */
    static getAbsoulteUrl(pageUrl: string, origUrl: string): string;
    static getHashCode(str: string, hashMethod: HashMethod): string;
    static convertToValidPathname(origPath: string, char?: string, isFilename?: boolean): string;
    static getUrlAttribute(urlStr: string, attrName: UrlAttrName | "hrefwithoutsearch" | "param", hostnameParts?: number, param?: string): string;
    /**
     * get the path of file, refer to template XXX
     * @param urlStr url, such as "https://aaa.bbb.company.com/AAA/BBB?p1=111&p2=222"
     * @param pathtype
     * @param hashMethod valid when pathtype ends with "ashcode"
     * @param hostnameParts default 0 that means all, how many last parts of hostname
     * * valid when pathtype includes "hostname"
     * * return "bbb.company.com" if hostnameParts is 3
     * @param paramsStr default "", params that are appended
     * * multiple params seperated by ","; such as "p1,p2", then "_111_222" will be appended
     * * ignored when pathtype includes "title"
     * @param extname default "", default extname if the original extname is not in validExtnames
     * * cannot be "" if pathtype includes "title", such as "pdf" or "mhtml"(because the original extname is "")
     * @param validExtnames default "", such as "pdf,mhtml", ignored when pathtype includes "title"
     * * if both extname and validExtnames are "" (defaut value), use the original extname
     * * else if original extname is in validExtnames, use the original extname
     * * else use the extname
     * @param title default "@"
     * @returns
     */
    static getFilePathFromUrl(urlStr: string, pathtype: PathType, hashMethod?: HashMethod, hostnameParts?: number, paramsStr?: string, extname?: string, validExtnames?: string, title?: string): string;
}

export { type FileType, GeneralFileLog, type GeneralFileLogOptions, type HashMethod, type LevelLogFunction, type LogFunction, LogLevel, LsdUrl, type PathType, type UrlAttrName, addLogFunction, addTime, checkDns, checkPing, compareTime, convertMapToObj, decodeFromBase64, encodeToBase64, execCommand, existsSync, fileExist, filesInDir, fsCheckOrCreateDir, fsCheckOrCreateSubdir, getComplementOfTwoArray, getCurrentUnixTime, getDaysOfMonth, getDaysOfYear, getDifferenceOfTwoArray, getDomainName, getHashCodeOfString, getHexStrOfIntNum, getIntersectionOfTwoArray, getJsonParseOfStr, getLocalDateNumber, getLocalDateString, getMemberOfObject, getMonthSecondsOfUnixTime, getNamesOfLogFunctions, getPerformanceOfPidTree, getPidsListeningOnPort, getRandomArrayIdx, getRandomInt, getRandomVcCode, getUTCDateString, getUnionOfTwoArray, getUniqueArray, getUnixTimeOfDay, getUnixTimeOfMonth, getUnixTimeOfWeek, getUnixTimeOfYear, getUtcUnixTimeOfDay, getUtcUnixTimeOfMonth, getUtcUnixTimeOfWeek, getUtcUnixTimeOfYear, getYyyyMmDdDateNum, isDirectory, isDirectorySync, isFile, isFileSync, log, mkdir, removeFile, removeLogFunction, setLogLevel, sleep, tryToAddDefaultFileLogFun, unreachable, unzipZippedBase64ToOrigStr, zipOrigStrToZippedBase64 };