@letsscrapedata/utils
Version:
This package is mainly used by LetsScrapeData App
1,137 lines (1,128 loc) • 32.7 kB
JavaScript
// src/utils/date.ts
function getCurrentUnixTime() {
return Math.floor(Date.now() / 1e3);
}
function getDaysOfYear(offset = 0) {
const d = /* @__PURE__ */ new Date();
const year = d.getFullYear() + offset;
return year % 4 === 0 && year % 100 !== 0 || year % 400 === 0 ? 366 : 365;
}
function getDaysOfMonth(offset = 0) {
const d = /* @__PURE__ */ new Date();
const year = d.getFullYear();
const month = d.getMonth();
return new Date(year, month + 1 + offset, 0).getDate();
}
function getLocalDateString(date = /* @__PURE__ */ new Date()) {
if (!(date instanceof Date)) {
return "";
}
const yyyy = date.getFullYear();
const MM = String(date.getMonth() + 1).padStart(2, "0");
const dd = String(date.getDate()).padStart(2, "0");
const HH = String(date.getHours()).padStart(2, "0");
const mm = String(date.getMinutes()).padStart(2, "0");
const ss = String(date.getSeconds()).padStart(2, "0");
return `${yyyy}-${MM}-${dd} ${HH}:${mm}:${ss}`;
}
function getUTCDateString(date = /* @__PURE__ */ new Date()) {
if (!(date instanceof Date)) {
return "";
}
return date.toISOString().slice(0, 19).replace("T", " ");
}
function getLocalDateNumber(date = /* @__PURE__ */ new Date()) {
if (!(date instanceof Date)) {
return 0;
}
const yyyy = date.getFullYear();
const MM = String(date.getMonth() + 1).padStart(2, "0");
const dd = String(date.getDate()).padStart(2, "0");
const HH = String(date.getHours()).padStart(2, "0");
const mm = String(date.getMinutes()).padStart(2, "0");
const ss = String(date.getSeconds()).padStart(2, "0");
return Number(`${yyyy}${MM}${dd}${HH}${mm}${ss}`);
}
function getMonthSecondsOfUnixTime(unixTime) {
const d = new Date(unixTime * 1e3);
const day = d.getDate();
const hour = d.getHours();
const minute = d.getMinutes();
const second = d.getSeconds();
return (((day - 1) * 24 + hour) * 60 + minute) * 60 + second;
}
function getUnixTimeOfDay(offset = 0) {
const d = /* @__PURE__ */ new Date();
const year = d.getFullYear();
const month = d.getMonth();
const day = d.getDate();
return Math.floor(new Date(year, month, day + offset).getTime() / 1e3);
}
function getUnixTimeOfWeek(offset = 0) {
const d = new Date(getUnixTimeOfDay() * 1e3);
const nextMonday = new Date(
d.setDate(
d.getDate() + ((7 - d.getDay() + 1) % 7 || 7)
)
);
return Math.floor(nextMonday.getTime() / 1e3) + (offset - 1) * 3600 * 24 * 7;
}
function getUnixTimeOfMonth(offset = 0) {
const d = /* @__PURE__ */ new Date();
const year = d.getFullYear();
const month = d.getMonth();
return Math.floor(new Date(year, month + offset, 0).getTime() / 1e3);
}
function getUnixTimeOfYear(offset = 0) {
const d = /* @__PURE__ */ new Date();
const year = d.getFullYear();
return Math.floor(new Date(year + offset, 0, 1).getTime() / 1e3);
}
function getUtcUnixTimeOfDay(offset = 0) {
const d = /* @__PURE__ */ new Date();
const year = d.getUTCFullYear();
const month = d.getUTCMonth();
const day = d.getUTCDate();
return Math.floor(new Date(year, month, day + offset).getTime() / 1e3);
}
function getUtcUnixTimeOfWeek(offset = 0) {
const d = new Date(getUtcUnixTimeOfDay() * 1e3);
const nextMonday = new Date(
d.setDate(
d.getDate() + ((7 - d.getDay() + 1) % 7 || 7)
)
);
return Math.floor(nextMonday.getTime() / 1e3) + (offset - 1) * 3600 * 24 * 7;
}
function getUtcUnixTimeOfMonth(offset = 0) {
const d = /* @__PURE__ */ new Date();
const year = d.getUTCFullYear();
const month = d.getUTCMonth();
return Math.floor(new Date(year, month + offset, 0).getTime() / 1e3);
}
function getUtcUnixTimeOfYear(offset = 0) {
const d = /* @__PURE__ */ new Date();
const year = d.getUTCFullYear();
return Math.floor(new Date(year + offset, 0, 1).getTime() / 1e3);
}
function getYyyyMmDdDateNum(date = /* @__PURE__ */ new Date(), diffDays = 0) {
if (!(date instanceof Date)) {
return 0;
}
const d = diffDays ? new Date(date.getTime() + diffDays * 24 * 3600 * 1e3) : date;
const year = d.getFullYear();
const month = d.getMonth() + 1;
const day = d.getDate();
return year * 1e4 + month * 100 + day;
}
function addTime(initialTime, addSeconds) {
return new Date(initialTime.getTime() + addSeconds * 1e3);
}
function compareTime(time1, time2) {
const num1 = time1.getTime();
const num2 = time2.getTime();
if (num1 > num2) {
return 1;
} else if (num1 < num2) {
return -1;
} else {
return 0;
}
}
// src/utils/file.ts
import fsp from "fs/promises";
import nodePath from "path";
import * as fs from "fs";
async function fileExist(path3) {
try {
await fsp.access(path3);
return true;
} catch {
return false;
}
}
async function filesInDir(path3, fileType = "general", prefix = "", extname = "") {
try {
const filenames = [];
const fileDirents = await fsp.readdir(path3, { withFileTypes: true });
fileDirents.forEach((fd) => {
switch (fileType) {
case "general":
if (!fd.isFile()) {
return;
}
break;
case "directory":
if (!fd.isDirectory()) {
return;
}
break;
case "block":
if (!fd.isBlockDevice()) {
return;
}
break;
case "character":
if (!fd.isCharacterDevice()) {
return;
}
break;
case "fifo":
if (!fd.isFIFO()) {
return;
}
break;
case "socket":
if (!fd.isSocket()) {
return;
}
break;
case "link":
if (!fd.isSymbolicLink()) {
return;
}
break;
case "any":
break;
default:
return;
}
if (prefix && !fd.name.startsWith(prefix)) {
return;
}
if (extname && !fd.name.endsWith(extname)) {
return;
}
filenames.push(fd.name);
});
return filenames;
} catch (err) {
return [];
}
}
async function isFile(path3) {
try {
const stats = await fsp.stat(path3);
const res = stats.isFile();
return res;
} catch (e) {
return false;
}
}
async function isDirectory(path3) {
try {
const stats = await fsp.stat(path3);
const res = stats.isDirectory();
return res;
} catch (e) {
return false;
}
}
async function mkdir(path3) {
try {
await fsp.mkdir(path3);
return true;
} catch {
return false;
}
}
async function fsCheckOrCreateDir(dirName, writeFlag = false, createFlag = false, recursiveFlag = false) {
recursiveFlag = false;
if (recursiveFlag) {
console.log(`${writeFlag}, ${recursiveFlag}`);
}
if (typeof dirName !== "string" || !dirName) {
return false;
}
let checkFlag = false;
checkFlag = await isDirectory(dirName);
if (checkFlag) {
return true;
}
if (!createFlag) {
return false;
} else {
const mkdirFlag = await mkdir(dirName);
return mkdirFlag;
}
}
async function fsCheckOrCreateSubdir(baseDir, subDir, sep = "") {
if (!sep) {
sep = nodePath.sep;
}
let checkDirFlag = await fsCheckOrCreateDir(baseDir, true, false);
if (!checkDirFlag) {
return false;
}
const trimSubDir = subDir.trim();
if (!trimSubDir) {
return true;
}
checkDirFlag = true;
let fullDir = baseDir;
const subDirs = trimSubDir.split(sep);
for (const dir of subDirs) {
fullDir = nodePath.join(fullDir, dir);
checkDirFlag = await fsCheckOrCreateDir(fullDir, true, true);
if (!checkDirFlag) {
return false;
}
}
return true;
}
async function removeFile(filename) {
try {
await fsp.unlink(filename);
return true;
} catch (err) {
return false;
}
}
function isDirectorySync(path3) {
try {
const stats = fs.statSync(path3);
return stats && stats.isDirectory();
} catch (e) {
return false;
}
}
function isFileSync(path3) {
try {
const stats = fs.statSync(path3);
return stats && stats.isFile();
} catch (e) {
return false;
}
}
function existsSync2(path3) {
try {
return fs.existsSync(path3);
} catch (e) {
return false;
}
}
// src/utils/log.ts
import path from "path";
import fs2 from "fs";
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
LogLevel2[LogLevel2["DBG"] = 0] = "DBG";
LogLevel2[LogLevel2["INF"] = 1] = "INF";
LogLevel2[LogLevel2["WRN"] = 2] = "WRN";
LogLevel2[LogLevel2["ERR"] = 3] = "ERR";
return LogLevel2;
})(LogLevel || {});
var logLevelNames = ["DBG", "INF", "WRN", "ERR"];
var logLevel = 2 /* WRN */;
var logFunsObj = {};
var logFuns = [];
function setLogLevel(newLevel) {
logLevel = newLevel;
return true;
}
function addLogFunction(name, logFunction) {
logFunsObj[name] = logFunction;
logFuns = Array.from(Object.values(logFunsObj));
return true;
}
function getNamesOfLogFunctions() {
return Array.from(Object.keys(logFunsObj));
}
function removeLogFunction(name) {
if (!logFunsObj[name]) {
return false;
}
delete logFunsObj[name];
logFuns = Array.from(Object.values(logFunsObj));
return true;
}
var log = async function log2(level, ...args) {
let ret = true;
if (level < logLevel) {
return ret;
}
for (const logFun of logFuns) {
try {
await logFun(level, ...args);
} catch (err) {
console.log(err);
ret = false;
}
}
return ret;
};
async function consoleLogFun(level, ...args) {
if (level === 0 /* DBG */) {
console.debug(...args);
} else if (level === 1 /* INF */) {
console.log(...args);
} else if (level === 2 /* WRN */) {
console.warn(...args);
} else if (level === 3 /* ERR */) {
console.error(...args);
} else {
return false;
}
return true;
}
var GeneralFileLog = class _GeneralFileLog {
#logFilenameDir;
#logFilenamePrefix;
#dateFormat;
#maxSize;
#intervals;
#datetimeIncluded;
#filename;
#fd;
#nextCheckTime;
#locked;
constructor(options = {}) {
const { logFilenameDir = "log", logFilenamePrefix = "lsd-", dateFormat = "yyyyMMdd-HHmmss", maxSize = 1024e4, intervals = 600, datetimeIncluded = true } = options;
this.#logFilenameDir = logFilenameDir;
this.#logFilenamePrefix = logFilenamePrefix;
this.#dateFormat = dateFormat;
this.#maxSize = maxSize;
this.#intervals = intervals;
this.#datetimeIncluded = datetimeIncluded;
this.#filename = "";
this.#fd = 0;
this.#nextCheckTime = getCurrentUnixTime() + intervals;
this.#locked = false;
}
/**
* get date string according to dateFormat
* @param dateFormat
* * yyyy: year such as "2024"
* * MM: month such as "01" or "12"
* * dd: day, such as "01" or "31"
* * HH: hour, such as "01" or "23"
* * mm: minute, such as "01" or "59"
* * ss: second, such as "01" or "59"
* @param date default new Date()
*/
static getDateString(dateFormat, date = /* @__PURE__ */ new Date()) {
let dateStr = dateFormat;
if (dateFormat.includes("yyyy")) {
const yyyy = date.getFullYear();
dateStr = dateStr.replaceAll("yyyy", String(yyyy));
}
if (dateFormat.includes("MM")) {
const MM = String(date.getMonth() + 1).padStart(2, "0");
dateStr = dateStr.replaceAll("MM", String(MM));
}
if (dateFormat.includes("dd")) {
const dd = String(date.getDate()).padStart(2, "0");
dateStr = dateStr.replaceAll("dd", String(dd));
}
if (dateFormat.includes("HH")) {
const HH = String(date.getHours()).padStart(2, "0");
dateStr = dateStr.replaceAll("HH", String(HH));
}
if (dateFormat.includes("mm")) {
const mm = String(date.getMinutes()).padStart(2, "0");
dateStr = dateStr.replaceAll("mm", String(mm));
}
if (dateFormat.includes("ss")) {
const ss = String(date.getSeconds()).padStart(2, "0");
dateStr = dateStr.replaceAll("ss", String(ss));
}
return dateStr;
}
#getLogFilename() {
if (this.#locked) {
return "";
}
this.#locked = true;
const dirFlag = isDirectorySync(this.#logFilenameDir);
if (!dirFlag) {
return "";
}
const dateStr = _GeneralFileLog.getDateString(this.#dateFormat);
let filename = `${this.#logFilenamePrefix}${dateStr}.log`;
const absoluteFlag = path.isAbsolute(this.#logFilenameDir);
if (absoluteFlag) {
filename = path.join(this.#logFilenameDir, filename);
} else {
const cwd = process.cwd();
filename = path.join(cwd, this.#logFilenameDir, filename);
}
const existFlag = fs2.existsSync(filename);
if (!existFlag) {
this.#locked = false;
return filename;
}
const fileFlag = isFileSync(filename);
this.#locked = false;
return fileFlag ? filename : "";
}
#checkLogfile() {
try {
if (!this.#fd) {
const filename = this.#getLogFilename();
if (filename) {
this.#filename = filename;
const fd = fs2.openSync(filename, "a");
this.#fd = fd;
return true;
} else {
return false;
}
}
const currentTime = getCurrentUnixTime();
if (this.#maxSize > 0 && this.#intervals > 0 && currentTime > this.#nextCheckTime) {
if (fs2.lstatSync(this.#filename).size >= this.#maxSize) {
fs2.closeSync(this.#fd);
const filename = this.#getLogFilename();
if (filename) {
this.#filename = filename;
const fd = fs2.openSync(filename, "a");
this.#fd = fd;
return true;
} else {
return false;
}
} else {
this.#nextCheckTime = currentTime + this.#intervals;
return true;
}
} else {
return true;
}
} catch (err) {
console.log(err);
return false;
}
}
static getMessage(level, args, datetimeIncluded = true) {
let msg = datetimeIncluded ? `${getLocalDateString()} ` : "";
msg += logLevelNames[level];
for (const arg of args) {
if (typeof arg === "string") {
msg += ` ${arg}
`;
} else if (arg instanceof Error) {
msg += ` Error stack: ${arg.stack}
`;
} else {
try {
msg += ` ${JSON.stringify(arg)}
`;
} catch (e) {
msg += " Cannot JSON.stringify\n";
}
}
}
return msg;
}
/**
* add a new log file
* * the directory must exist, or return false!
* * if the file exists, it must be a general text file and you have write privilege
* * if the file does not exist, it will be created
* @param path
*/
async log(level, ...args) {
try {
const checkFlag = this.#checkLogfile();
if (!checkFlag) {
return false;
}
const msg = _GeneralFileLog.getMessage(level, args, this.#datetimeIncluded);
fs2.appendFileSync(this.#fd, msg, "utf8");
return true;
} catch (err) {
console.log(err);
return false;
}
}
};
function tryToAddDefaultFileLogFun() {
const logDirExists = isDirectorySync("log");
if (logDirExists) {
const defaultFileLog = new GeneralFileLog();
addLogFunction("defaultFile", defaultFileLog.log.bind(defaultFileLog));
}
return logDirExists;
}
addLogFunction("console", consoleLogFun);
// src/utils/math.ts
function getRandomInt(minNum, maxNum) {
if (typeof minNum !== "number" || typeof maxNum !== "number" || minNum >= maxNum) {
return 0;
} else {
return Number(minNum + (maxNum - minNum) * Math.random());
}
}
function getHexStrOfIntNum(intNum, strNum = 16) {
return intNum.toString(16).padStart(strNum, "0");
}
function getComplementOfTwoArray(array1, array2) {
if (!Array.isArray(array1) || !Array.isArray(array2)) {
return [];
}
const union = new Set(array1.concat(array2));
const set1 = new Set(array1);
const set2 = new Set(array2);
const resultArr = [];
for (let i of union) {
if (!(set1.has(i) && set2.has(i))) {
resultArr.push(i);
}
}
return resultArr;
}
function getDifferenceOfTwoArray(array1, array2) {
if (!Array.isArray(array1) || !Array.isArray(array2)) {
return [];
}
const set1 = new Set(array1);
const set2 = new Set(array2);
const resultArr = [];
for (let i of set1) {
if (!set2.has(i)) {
resultArr.push(i);
}
}
return resultArr;
}
function getIntersectionOfTwoArray(array1, array2) {
if (!Array.isArray(array1) || !Array.isArray(array2)) {
return [];
}
const set1 = new Set(array1);
const set2 = new Set(array2);
const resultArr = [];
for (let i of set1) {
if (set2.has(i)) {
resultArr.push(i);
}
}
return resultArr;
}
function getUnionOfTwoArray(array1, array2) {
if (!Array.isArray(array1) || !Array.isArray(array2)) {
return [];
}
const resultArr = Array.from(new Set(array1.concat(array2)));
return resultArr;
}
// src/utils/misc.ts
async function sleep(miliseconds) {
return new Promise((resolve) => {
setTimeout(resolve, miliseconds);
});
}
function unreachable(x) {
console.log(x);
throw new Error("Didn't expect to get here");
}
var vcCodeStr = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
function getRandomVcCode(vcCodeLength = 6, strScopeLen = 10) {
let vcCode = "";
for (let i = 0; i < vcCodeLength; i++) {
vcCode += vcCodeStr.charAt(Math.floor(Math.random() * strScopeLen));
}
return vcCode;
}
function getMemberOfObject(origObj, key1, key2 = "", key3 = "", key4 = "", key5 = "", key6 = "") {
let retObj = origObj;
if (!key1) {
return retObj;
}
if (typeof retObj !== "object" || !retObj) {
return null;
} else {
retObj = retObj[key1];
}
if (!key2) {
return retObj;
}
if (typeof retObj !== "object" || !retObj) {
return null;
} else {
retObj = retObj[key2];
}
if (!key3) {
return retObj;
}
if (typeof retObj !== "object" || !retObj) {
return null;
} else {
retObj = retObj[key3];
}
if (!key4) {
return retObj;
}
if (typeof retObj !== "object" || !retObj) {
return null;
} else {
retObj = retObj[key4];
}
if (!key5) {
return retObj;
}
if (typeof retObj !== "object" || !retObj) {
return null;
} else {
retObj = retObj[key5];
}
if (!key6) {
return retObj;
}
if (typeof retObj !== "object" || !retObj) {
return null;
} else {
retObj = retObj[key6];
}
return retObj;
}
function getJsonParseOfStr(str, defaultVal = {}) {
if (typeof str !== "string" || !str) {
return defaultVal;
}
try {
return JSON.parse(str);
} catch (err) {
return defaultVal;
}
}
function getRandomArrayIdx(length) {
if (length <= 0 || typeof length !== "number") {
return [];
}
return [...Array(length).keys()].sort(() => Math.random() - 0.5);
}
function getUniqueArray(origArray) {
return [...new Set(origArray)];
}
function convertMapToObj(map) {
if (map instanceof Map) {
const obj = {};
for (const [key, value] of map.entries()) {
obj[key] = value;
}
return obj;
} else {
return map;
}
}
// src/utils/network.ts
import dns from "dns";
import ping from "ping";
async function checkDns(domainName) {
try {
await dns.promises.lookup(domainName);
return true;
} catch (err) {
return false;
}
}
async function checkPing(host) {
const res = await ping.promise.probe(host);
return res && res.alive ? true : false;
}
async function getDomainName(url) {
const urlObj = new URL(url);
const parts = urlObj.hostname.toLowerCase().split(".");
for (let i = parts.length - 2; i >= 0; i--) {
const domain = parts.slice(i).join(".");
const validFlag = await checkDns(domain);
if (validFlag) {
return domain;
}
}
return "";
}
// src/utils/string.ts
import zlib from "zlib";
function decodeFromBase64(base64Str) {
try {
const jsonStr = Buffer.from(base64Str, "base64").toString("utf8");
const actualData = JSON.parse(jsonStr);
return actualData;
} catch (err) {
log(3 /* ERR */, err);
return null;
}
}
function encodeToBase64(origData) {
try {
const jsonStr = JSON.stringify(origData);
const base64Str = Buffer.from(jsonStr, "utf8").toString("base64");
return base64Str;
} catch (err) {
log(3 /* ERR */, err);
return "";
}
}
async function zipOrigStrToZippedBase64(origStr) {
return new Promise((resolve) => {
zlib.gzip(origStr, (err, buffer) => {
if (err) {
resolve("");
} else {
resolve(buffer.toString("base64"));
}
});
});
}
async function unzipZippedBase64ToOrigStr(zippedBase64) {
return new Promise(
(resolve) => zlib.gunzip(Buffer.from(zippedBase64, "base64"), (err, buffer) => {
if (err) {
resolve("");
} else {
resolve(buffer.toString());
}
})
);
}
function getHashCodeOfString(str, divisor = 1024) {
let hash = 0;
let chr = 0;
if (!str)
return hash;
for (let i = 0; i < str.length; i++) {
chr = str.charCodeAt(i);
hash = (hash << 5) - hash + chr;
hash |= 0;
}
return Math.abs(hash) % divisor;
}
// src/utils/system.ts
import { exec } from "child_process";
import pidusageTree from "pidusage-tree";
async function execCommand(command) {
const promise = new Promise((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
if (error) {
console.log(error);
}
if (stderr) {
reject(stderr);
} else {
resolve(stdout);
}
});
});
const retStr = await promise;
return retStr;
}
async function getPidsListeningOnPort(port, protocol = "TCP") {
try {
const platform = process.platform;
const command = platform === "win32" ? `netstat -ano` : `lsof -i :${port}`;
const retStr = await execCommand(command);
let pids = [];
if (platform === "win32") {
const reStr = protocol === "TCP" ? `^\\s*TCP.*:${port}\\s+.*\\s+LISTENING\\s+` : `^\\s*UDP.*:${port}\\s+`;
const re = new RegExp(reStr);
const lines = retStr.trim().split(/[\n\r]+/).filter((line) => line.match(re));
const allPids = lines.map((line) => parseInt(line.split(/\s+/).slice(-1)[0]));
pids = Array.from(new Set(allPids));
} else {
const re = protocol === "TCP" ? /\s+TCP\s+/ : /\s+UDP\s+/;
const lines = retStr.trim().split(/[\n\r]+/).filter((line) => line.match(re));
const allPids = lines.map((line) => parseInt(line.split(/\s+/)[1]));
pids = Array.from(new Set(allPids));
}
return pids;
} catch (err) {
throw err;
}
}
async function getPerformanceOfPidTree(pid, memoryUnit = "bytes") {
try {
const tree = await pidusageTree(pid);
const processes = Array.from(Object.values(tree));
const cpu = processes.reduce((sum, p) => sum + p.cpu, 0);
let memory = processes.reduce((sum, p) => sum + p.memory, 0);
if (memoryUnit === "MB") {
memory = Math.round(memory / 1024 / 1024);
} else if (memoryUnit === "GB") {
memory = Math.round(memory / 1024 / 1024 / 1024);
}
return { cpu, memory };
} catch (err) {
return { cpu: 0, memory: 0 };
}
}
// src/utils/url.ts
import path2 from "path";
import { createHash } from "crypto";
import * as uuid from "uuid";
var LsdUrl = class _LsdUrl {
static #getPathName(pathname) {
return pathname.split("/").slice(1).join("_");
}
static #getHostnameParts(hostname, partsNum) {
if (partsNum > 0) {
return hostname.split(".").slice(0, partsNum).join(".");
} else if (partsNum < 0) {
return hostname.split(".").slice(partsNum).join(".");
} else {
return hostname;
}
}
/**
* Get absolute url of oriUrl
** return origUrl if origUrl starts with "http"
* @param pageUrl
* @param origUrl
* @returns return "" if invalid paras or failed
*/
static getAbsoulteUrl(pageUrl, origUrl) {
try {
if (!pageUrl || !origUrl) {
return "";
}
const prefix = origUrl.slice(0, 7).toLowerCase();
if (prefix.startsWith("http")) {
return origUrl;
}
const url = new URL(origUrl, pageUrl);
return url ? url.href : origUrl;
} catch (err) {
return "";
}
}
static getHashCode(str, hashMethod) {
try {
if (typeof str !== "string" || typeof hashMethod !== "string" || !hashMethod) {
return "";
} else if (hashMethod === "uuidv1") {
return uuid.v1();
} else if (hashMethod === "uuidv4") {
return uuid.v4();
} else {
return createHash(hashMethod).update(str).digest("hex");
}
} catch (err) {
return str;
}
}
static convertToValidPathname(origPath, char = "#", isFilename = true) {
try {
if (isFilename) {
return origPath.replace(/[\\/:*?'"<>|]/g, char);
} else {
return origPath.replace(/[:*?'"<>|]/g, char);
}
} catch (err) {
return "";
}
}
static #getFinalExtname(origExtname, defaultExtname, validExtnamesStr) {
if (!defaultExtname && !validExtnamesStr) {
return origExtname;
}
let newExtname = origExtname;
if (origExtname && validExtnamesStr) {
const validExtnames = validExtnamesStr.split(",");
newExtname = validExtnames.includes(origExtname) ? origExtname : defaultExtname;
} else {
newExtname = defaultExtname;
}
return newExtname;
}
// attrName: "pathname", "href", "origin", "protocol", "username", "password", "host", "search", "hash", "hostname", "port", "hrefwithoutsearch", "param"
static getUrlAttribute(urlStr, attrName, hostnameParts = 0, param = "") {
try {
let ret = "";
const url = new URL(urlStr);
if (["href", "origin", "protocol", "username", "password", "host", "pathname", "search", "hash"].includes(attrName)) {
ret = url[attrName];
} else if (attrName === "hostname") {
ret = _LsdUrl.#getHostnameParts(url.hostname, hostnameParts);
} else if (attrName === "port") {
const protocol = url.protocol;
let port = url.port;
if (!port && protocol === "http:") {
port = "80";
} else if (!port && protocol === "https:") {
port = "443";
}
ret = port;
} else if (attrName === "param") {
const val = url["searchParams"].get(param);
ret = val ? val : "";
} else if (attrName === "hrefwithoutsearch") {
ret = url.port ? `${url.protocol}//${url.hostname}::${url.port}/${url.pathname}` : `${url.protocol}//${url.hostname}/${url.pathname}`;
} else {
log(3 /* ERR */, `Invalid pathtype ${attrName} in getUrlAttribute`);
return "";
}
return ret ? ret : "";
} catch (err) {
log(3 /* ERR */, err);
return "";
}
}
/**
* get the path of file, refer to template XXX
* @param urlStr url, such as "https://aaa.bbb.company.com/AAA/BBB?p1=111&p2=222"
* @param pathtype
* @param hashMethod valid when pathtype ends with "ashcode"
* @param hostnameParts default 0 that means all, how many last parts of hostname
* * valid when pathtype includes "hostname"
* * return "bbb.company.com" if hostnameParts is 3
* @param paramsStr default "", params that are appended
* * multiple params seperated by ","; such as "p1,p2", then "_111_222" will be appended
* * ignored when pathtype includes "title"
* @param extname default "", default extname if the original extname is not in validExtnames
* * cannot be "" if pathtype includes "title", such as "pdf" or "mhtml"(because the original extname is "")
* @param validExtnames default "", such as "pdf,mhtml", ignored when pathtype includes "title"
* * if both extname and validExtnames are "" (defaut value), use the original extname
* * else if original extname is in validExtnames, use the original extname
* * else use the extname
* @param title default "@"
* @returns
*/
static getFilePathFromUrl(urlStr, pathtype, hashMethod = "MD5", hostnameParts = 0, paramsStr = "", extname = "", validExtnames = "", title = "@") {
try {
let ret = "";
if (!urlStr.slice(0, 10).toLowerCase().startsWith("http")) {
ret = _LsdUrl.getHashCode(urlStr, hashMethod);
return extname ? `${ret}.${extname}` : ret;
}
const url = new URL(urlStr);
if (pathtype === "basename") {
ret = path2.basename(url.pathname);
} else if (pathtype === "hostnamePathname") {
const hostname = _LsdUrl.#getHostnameParts(url.hostname, hostnameParts);
ret = `${hostname}${url.pathname}`;
} else if (pathtype === "hostnamePath_name") {
const hostname = _LsdUrl.#getHostnameParts(url.hostname, hostnameParts);
ret = `${hostname}/${_LsdUrl.#getPathName(url.pathname)}`;
} else if (pathtype === "hostname_path_name") {
const hostname = _LsdUrl.#getHostnameParts(url.hostname, hostnameParts);
ret = `${hostname}_${_LsdUrl.#getPathName(url.pathname)}`;
} else if (pathtype === "pathname") {
ret = url.pathname;
} else if (pathtype === "path_name") {
ret = _LsdUrl.#getPathName(url.pathname);
} else if (pathtype.endsWith("ashcode")) {
const hostname = _LsdUrl.#getHostnameParts(url.hostname, hostnameParts);
const hashCode = _LsdUrl.getHashCode(url.toString(), hashMethod);
let extname2 = path2.extname(url.pathname);
if (pathtype === "hashcode") {
ret = `${hashCode}${extname2}`;
} else if (pathtype === "hostnameHashcode") {
ret = `${hostname}/${hashCode}${extname2}`;
} else if (pathtype === "hostname_hashcode") {
ret = `${hostname}_${hashCode}${extname2}`;
} else {
log(3 /* ERR */, `Invalid pathtype ${pathtype} in getFilePathFromUrl`);
return "";
}
} else if (pathtype.endsWith("itle")) {
const newTitle = _LsdUrl.convertToValidPathname(title);
if (pathtype === "hostnameTitle" || pathtype === "hostname_title") {
const hostname = _LsdUrl.#getHostnameParts(url.hostname, hostnameParts);
if (pathtype === "hostnameTitle") {
ret = newTitle ? `${hostname}/${newTitle}` : `${hostname}/@`;
} else {
ret = `${hostname}_${newTitle}`;
}
} else if (pathtype === "title") {
ret = newTitle ? newTitle : "@";
} else {
log(3 /* ERR */, `Invalid pathtype ${pathtype} in getFilePathFromUrl`);
return "";
}
return extname ? `${ret}.${extname}` : ret;
} else {
log(3 /* ERR */, `Invalid pathtype ${pathtype} in getFilePathFromUrl`);
return "";
}
let origExtname = path2.extname(url.pathname).slice(1);
const len = origExtname.length;
if (len > 0) {
ret = ret.slice(0, -(len + 1));
}
if (paramsStr) {
const params = paramsStr.split(",");
let paramsValue = "";
params.forEach((param) => {
let val = url.searchParams.get(param);
val = val ? val : "";
paramsValue = paramsValue ? `${paramsValue}_${val}` : val;
});
paramsValue = _LsdUrl.convertToValidPathname(paramsValue);
ret = `${ret}_${paramsValue}`;
}
const newExtname = _LsdUrl.#getFinalExtname(origExtname, extname, validExtnames);
if (newExtname) {
ret = `${ret}.${newExtname}`;
}
return ret ? ret : "";
} catch (err) {
log(3 /* ERR */, err);
return "";
}
}
};
export {
GeneralFileLog,
LogLevel,
LsdUrl,
addLogFunction,
addTime,
checkDns,
checkPing,
compareTime,
convertMapToObj,
decodeFromBase64,
encodeToBase64,
execCommand,
existsSync2 as existsSync,
fileExist,
filesInDir,
fsCheckOrCreateDir,
fsCheckOrCreateSubdir,
getComplementOfTwoArray,
getCurrentUnixTime,
getDaysOfMonth,
getDaysOfYear,
getDifferenceOfTwoArray,
getDomainName,
getHashCodeOfString,
getHexStrOfIntNum,
getIntersectionOfTwoArray,
getJsonParseOfStr,
getLocalDateNumber,
getLocalDateString,
getMemberOfObject,
getMonthSecondsOfUnixTime,
getNamesOfLogFunctions,
getPerformanceOfPidTree,
getPidsListeningOnPort,
getRandomArrayIdx,
getRandomInt,
getRandomVcCode,
getUTCDateString,
getUnionOfTwoArray,
getUniqueArray,
getUnixTimeOfDay,
getUnixTimeOfMonth,
getUnixTimeOfWeek,
getUnixTimeOfYear,
getUtcUnixTimeOfDay,
getUtcUnixTimeOfMonth,
getUtcUnixTimeOfWeek,
getUtcUnixTimeOfYear,
getYyyyMmDdDateNum,
isDirectory,
isDirectorySync,
isFile,
isFileSync,
log,
mkdir,
removeFile,
removeLogFunction,
setLogLevel,
sleep,
tryToAddDefaultFileLogFun,
unreachable,
unzipZippedBase64ToOrigStr,
zipOrigStrToZippedBase64
};