kaven-utils
Version:
Utils for Node.js.
692 lines (691 loc) • 27.4 kB
JavaScript
/********************************************************************
* @author: Kaven
* @email: kaven@wuwenkai.com
* @website: http://blog.kaven.xyz
* @file: [Kaven-Utils] /src/KavenUtility.Net.ts
* @create: 2018-08-30 11:26:09.057
* @modify: 2025-10-22 15:20:06.837
* @version: 6.1.1
* @times: 247
* @lines: 793
* @copyright: Copyright © 2018-2025 Kaven. All Rights Reserved.
* @description: [description]
* @license: [license]
********************************************************************/
import pkgFollowRedirects from "follow-redirects";
import { ArrayRemove, ConvertTo, GetBaseDir, GetFileNameFromURL, GetStringBetween, HttpStatusCode, IsEqual, IsHttpsUrl, IsSuccessStatusCode, KavenCache, Sleep, Strings_CR_LF, Strings_DoubleQuotes, Strings_Empty } from "kaven-basic";
import { createSocket } from "node:dgram";
import { createWriteStream, existsSync, unlinkSync } from "node:fs";
import { isIP } from "node:net";
import { networkInterfaces } from "node:os";
import { KavenRequest } from "./KavenRequest.js";
import { MakeDirectory, MakeDirectorySync } from "./KavenUtility.FileSystem.js";
import { GetFileType } from "./KavenUtility.ThirdParty.js";
import { DefaultGetExternalIpByUPnPOptions } from "./base/Constants.js";
/**
* @since 5.4.0
* @version 2024-11-01
*/
export async function HttpGet(url, timeout) {
const request = new KavenRequest(url);
const r = await request.Execute(timeout);
return r;
}
/**
* Makes an asynchronous attempt to perform a GET request to one or more URLs.
* If multiple URLs are provided, it resolves with the first successful response.
* @param urls - A single URL or an array of URLs to attempt the GET request.
* @returns A promise that resolves with the first successful response or `undefined` if all attempts fail.
* @since 5.4.0
* @version 2025-10-14
*/
export async function HttpTryGetText(urls, timeout, logger) {
try {
// If a single URL is provided, convert it to an array for uniform handling.
if (typeof urls === "string") {
urls = [urls];
}
let n = urls.length;
// If no URLs are provided, return undefined.
if (n === 0) {
return undefined;
}
// If only one URL is provided, directly call HttpGet and return its result.
else if (n === 1) {
const r = await HttpGet(urls[0], timeout);
if (r.IsSuccess) {
return r.Text;
}
return undefined;
}
// If multiple URLs are provided, attempt GET requests in parallel.
return new Promise((resolve) => {
for (const item of urls) {
// Perform a GET request for each URL.
HttpGet(item, timeout).then(result => {
// If a successful response is obtained, resolve the main promise.
if (result.IsSuccess && result.Text) {
resolve(result.Text);
}
else {
// If a response is not obtained, decrement the counter.
if (--n === 0) {
// If all attempts fail, resolve with undefined.
resolve(undefined);
}
}
}).catch(ex => {
// Log any internal errors but continue trying other URLs.
logger?.Error(ex);
// 2024-05-23
// If a response is not obtained, decrement the counter.
if (--n === 0) {
// If all attempts fail, resolve with undefined.
resolve(undefined);
}
});
}
});
}
catch (ex) {
logger?.Error(ex);
return undefined;
}
}
/**
* Makes an asynchronous attempt to perform a GET request to one or more URLs.
* If multiple URLs are provided, it resolves with the first successful response.
* @param urls - A single URL or an array of URLs to attempt the GET request.
* @returns A promise that resolves with the first successful response or `undefined` if all attempts fail.
* @since 5.4.0
* @version 2025-10-14
*/
export async function HttpTryGetJson(urls, timeout, logger) {
try {
const r = await HttpTryGetText(urls, timeout, logger);
if (!r) {
return undefined;
}
return JSON.parse(r);
}
catch (ex) {
logger?.Error(ex);
return undefined;
}
}
/**
* @since 1.1.23
* @version 2025-05-22
*/
export async function HttpPost(url, data) {
const request = new KavenRequest(url);
request.RequestData = data;
return request.Execute();
}
/**
* @since 1.0.9
* @version 2025-10-14
*/
export async function GetExternalIp(options) {
options ??= {};
const logger = options.logger;
if (options.rules && options.rules.length > 0) {
for (const rule of options.rules) {
try {
const data = await HttpTryGetText(rule.url, 10000);
if (data) {
if (rule.rule) {
const ip = rule.rule(data);
if (isIP(ip)) {
return ip;
}
}
if (rule.regex) {
const re = new RegExp(rule.regex.pattern, rule.regex.flags);
const match = data.match(re);
if (match) {
if (rule.regex.resultIndex !== undefined) {
const ip = match[rule.regex.resultIndex];
if (isIP(ip)) {
return ip;
}
}
else {
for (const m of match) {
if (isIP(m)) {
return m;
}
}
}
}
}
if (isIP(data)) {
return data;
}
const ip = data.match(/\d+\.\d+\.\d+\.\d+/)?.[0];
if (ip && isIP(ip)) {
return ip;
}
}
logger?.Warn(`GetExternalIp failed, url:${rule.url}, data:${data}`);
}
catch (ex) {
logger?.Error(ex);
}
}
if (options.ignoreBuiltInRules) {
/* istanbul ignore next */
return Strings_Empty;
}
}
let ip = Strings_Empty;
const key = "GetExternalIp_Rules";
let rules = KavenCache.GetValue(key);
// Retrieve from server
if (!rules || rules.length === 0) {
rules = await HttpTryGetJson([
"https://public.data.wuwenkai.com/json/get-external-ip.json",
"https://public.data.kaven.xyz/json/get-external-ip.json",
]);
logger?.Info(`Retrieve ${rules?.length} rules from server`);
}
// Default fallback
if (!rules || rules.length === 0) {
rules = [
{
url: "https://ip.wuwenkai.com",
},
{
url: "https://ip.kaven.xyz",
},
];
}
const failedItems = [];
for (const item of rules) {
if (options.ipv6 !== undefined) {
if (options.ipv6) {
if (item.type !== "IPv6") {
continue;
}
}
else {
if (item.type === "IPv6") {
continue;
}
}
}
logger?.Info(`Try get external IP address from: ${item.url}`);
if (!item.url) {
continue;
}
const result = await GetExternalIp({
rules: [item],
ignoreBuiltInRules: true,
});
if (result) {
ip = result;
break;
}
failedItems.push(item);
}
if (failedItems.length > 0) {
ArrayRemove(rules, failedItems);
rules.push(...failedItems);
}
if (ip) {
KavenCache.SetValue(key, rules);
}
else {
KavenCache.RemoveValue(key);
}
/* istanbul ignore next */
return ip; // 2021-03-06
}
/**
* @since 1.0.9
* @version 2025-10-14
*/
export async function DownloadFile(url, savePath, options) {
options ??= {};
await MakeDirectory(GetBaseDir(savePath));
const httpGet = pkgFollowRedirects.http.get;
const httpsGet = pkgFollowRedirects.https.get;
return new Promise((resolve, reject) => {
try {
const file = createWriteStream(savePath);
file.on("error", (err) => {
options.logger?.Error(err);
});
const get = IsHttpsUrl(url) ? httpsGet : httpGet;
get(url, options.http, (response) => {
response.pipe(file);
file.on("finish", () => {
file.close();
return resolve(savePath);
});
}).on("error", (err) => {
// Handle errors
if (existsSync(savePath)) {
unlinkSync(savePath);
return reject(err);
}
});
}
catch (ex) {
return reject(ex);
}
});
}
/**
*
* @param url
* @param options
* @param tryDetectFileTypeByMagicNumber
* @param minSizeInBytes
* @since 4.0.0
* @version 2022-04-09
* @returns
*/
export async function HttpDownload(url, options, tryDetectFileTypeByMagicNumber = true, minSizeInBytes) {
const { http, https } = await import("follow-redirects");
const httpGet = http.get;
const httpsGet = https.get;
const isHttps = IsHttpsUrl(url);
const get = isHttps ? httpsGet : httpGet;
const ContentDisposition = "content-disposition";
// const ContentLength = "content-length";
return new Promise((resolve, reject) => {
const req = get(url, res => {
const { statusCode, statusMessage, headers } = res;
if (statusCode !== HttpStatusCode.OK) {
return reject(new Error(`Request failed with ${statusMessage}[${statusCode}]`));
}
if (!options.Folder) {
return reject(new Error("Folder is empty."));
}
// if (minSizeInBytes) {
// const fileSize = Number(headers[ContentLength]);
// if (fileSize) {
// // InternalLogger.Info(`file size: ${fileSize}`.AsLog());
// if (fileSize < minSizeInBytes) {
// return (new Error(`File too small: ${fileSize} < ${minSizeInBytes}.`));
// } else {
// minSizeInBytes = undefined;
// }
// } else {
// InternalLogger.Warn(BuildLogString(LogLevel.Warn, "Cannot read file size from http headers."));
// }
// }
if (!options.Name) {
const contentDispositionHeader = ConvertTo(headers[ContentDisposition]);
if (contentDispositionHeader) {
// Content-Disposition: form-data
// Content-Disposition: form-data; name="fieldName"
// Content-Disposition: form-data; name="fieldName"; filename="filename.jpg"
const parameters = contentDispositionHeader.split(";");
let temp = parameters.find((p) => p.startsWith("filename"));
if (temp) {
options.Name = GetStringBetween(temp, Strings_DoubleQuotes, Strings_DoubleQuotes, 1)[0];
}
if (!options.Name) {
temp = parameters.find((p) => p.startsWith("name"));
if (temp) {
options.Name = GetStringBetween(temp, Strings_DoubleQuotes, Strings_DoubleQuotes, 1)[0];
}
}
}
if (!options.Name) {
options.NameWithExtension = GetFileNameFromURL(url);
}
}
if (options.Folder) {
MakeDirectorySync(options.Folder);
}
if (tryDetectFileTypeByMagicNumber) {
// response.data.on("readable", () => {
// const chunk = response.data.read(16);
// if (chunk) {
// // InternalLogger.Info("got %d bytes of data", chunk.length);
// const fileType = KavenUtility.GetFileType(chunk);
// if (fileType && fileType.ext) {
// newFilePath = KavenUtility.ChangeFileExtension(options.FilePath, fileType.ext);
// }
// }
// });
let writeStream;
let buffer;
let lastTime = Date.now();
const timer = setInterval(() => {
if (Date.now() - lastTime > 5000) {
clearInterval(timer);
writeStream?.close();
reject(new Error("Timeout"));
}
}, 5000);
const Reject = (err) => {
clearInterval(timer);
if (writeStream !== undefined) {
writeStream.close();
}
reject(err);
};
const WriteData = (data) => {
lastTime = Date.now();
if (writeStream !== undefined) {
writeStream.write(data, err => {
if (err) {
return Reject(err);
}
lastTime = Date.now();
});
}
else {
GetFileType(ConvertTo(data)).then(fileType => {
if (fileType && fileType.ext) {
options.Extension = fileType.ext;
}
writeStream = createWriteStream(options.DynamicPath);
writeStream.on("close", () => {
if (minSizeInBytes) {
return Reject(new Error(`File too small: ${buffer ? buffer.length : 0} < ${minSizeInBytes}.`));
}
return resolve(options);
}).on("error", (err) => {
return Reject(err);
});
WriteData(data);
}).catch(err => {
return Reject(err);
});
}
};
res.on("data", (chunk) => {
if (chunk) {
// InternalLogger.Info("got %d bytes of data", chunk.length);
if (minSizeInBytes) {
if (buffer === undefined) {
buffer = Buffer.from(ConvertTo(chunk));
}
else {
buffer = Buffer.concat([buffer, Buffer.from(ConvertTo(chunk))]);
}
if (buffer.length > minSizeInBytes) {
WriteData(buffer);
minSizeInBytes = undefined;
}
}
else {
WriteData(chunk);
}
}
});
res.on("end", () => {
process.nextTick(() => {
if (writeStream !== undefined) {
writeStream.end();
}
else {
if (minSizeInBytes) {
return Reject(new Error(`File too small: ${buffer ? buffer.length : 0} < ${minSizeInBytes}.`));
}
return Reject(new Error());
}
});
});
res.on("error", (err) => {
return Reject(err);
});
}
else {
// pipe the result stream into a file on disc
res.pipe(createWriteStream(options.DynamicPath))
.on(/* "finish" */ "close", () => {
// InternalLogger.Info("finished");
return resolve(options);
}).on("error", (err) => {
return reject(err);
});
}
});
req.on("error", (e) => {
reject(e);
});
});
}
/**
* @since 5.0.4
* @version 2023-11-25
*/
export async function ExecuteSoapAction(options) {
const { url, serviceType, action, actionResultName, logger } = options;
const soapAction = `"${serviceType}#${action}"`;
const data = "<s:Envelope" +
" xmlns:s=\"http://schemas.xmlsoap.org/soap/envelope/\"" +
" s:encodingStyle=\"http://schemas.xmlsoap.org/soap/encoding/\">" +
" <s:Body>" +
` <u:${action} xmlns:u="${serviceType}">` +
` </u:${action}>` +
" </s:Body>" +
"</s:Envelope>";
const request = new KavenRequest(url);
request.RequestData = data;
request.UpdateHeader("Content-Type", "text/xml; charset=\"utf-8\"");
request.UpdateHeader("SOAPAction", soapAction);
const r = await request.Execute();
if (!IsSuccessStatusCode(r.Status)) {
logger?.Warn(`[ExecuteSoapAction][${r.StatusText ?? r.Status}] ${url}, ${serviceType}, ${action}, ${r.Text}`);
return;
}
if (r.Text && actionResultName) {
const result = GetStringBetween(r.Text, `<${actionResultName}>`, `</${actionResultName}>`, 1)[0];
return result;
}
return r.Text;
}
/**
* @since 5.0.3
* @version 2025-10-14
*/
export async function GetExternalIpByUPnP(parameters) {
const options = {
...DefaultGetExternalIpByUPnPOptions,
...parameters,
};
const logger = options.logger;
const getFromLocation = async (location) => {
const r = await HttpGet(location);
if (!IsSuccessStatusCode(r.Status)) {
logger?.Warn(`[${r.StatusText ?? r.Status}] ${location}, ${r.Text}`);
return;
}
if (!r.Text) {
return;
}
const ip = GetStringBetween(r.Text, "<NewExternalIPAddress>", "</NewExternalIPAddress>", 1)[0];
return {
ip,
data: r.Text,
};
};
const cacheKey = "GetExternalIpByUPnP_Cache";
if (options.ignoreCache !== true) {
try {
const cache = KavenCache.GetValue(cacheKey);
if (cache !== undefined) {
if (typeof cache === "string") {
const r = await getFromLocation(cache);
if (options.checkIp(r?.ip)) {
return r?.ip;
}
}
else {
const ip = await ExecuteSoapAction(cache);
if (options.checkIp(ip)) {
return ip;
}
}
}
}
catch (ex) {
logger?.Warn(ex);
KavenCache.RemoveValue(cacheKey);
}
}
// Start the timer
const startTime = process.hrtime();
/**
* SSDP M-SEARCH message
* HOST: Specifies the multicast address and port.
* MAN: Specifies the discovery method. In this case, it's set to "ssdp:discover."
* MX: Specifies the maximum wait time in seconds for responses.
* ST: Specifies the search target. "ssdp:all" means the device is searching for all types of devices.
*/
const searchMessage = [
"M-SEARCH * HTTP/1.1",
`HOST: ${options.multicastIp}:${options.port}`,
"MAN: \"ssdp:discover\"",
`MX: ${Math.max(1, Math.floor(options.timeoutMilliseconds / 1000))}`,
"ST: ssdp:all",
Strings_Empty,
].join(Strings_CR_LF) + Strings_CR_LF;
const socketList = [];
try {
const messageList = [];
const locationList = [];
// Get a list of network interfaces
const interfaces = networkInterfaces();
// Iterate over each network interface
Object.keys(interfaces).forEach((interfaceName) => {
const interfaceInfo = interfaces[interfaceName];
// Iterate over each address in the interface
interfaceInfo?.forEach((addressInfo) => {
// Check if it's an IPv4 address and not a loopback address
if (addressInfo.family === "IPv4" && !addressInfo.internal && addressInfo) {
// Create a UDP socket for each interface and bind to the address
const socket = createSocket("udp4");
socketList.push(socket);
socket.on("listening", () => {
socket.setBroadcast(true);
socket.setMulticastTTL(128);
socket.addMembership(options.multicastIp);
});
// Listen for responses (optional)
socket.on("message", (msg, rinfo) => {
const message = msg.toString();
logger?.Info(`Received response from ${rinfo.address}:${rinfo.port} on interface ${interfaceName}: ${message}`);
messageList.push(message);
});
// Handle socket errors
socket.on("error", (err) => {
logger?.Error(`Socket error on interface ${interfaceName}: ${err}`);
});
// Handle socket close
socket.on("close", () => {
logger?.Info(`Socket closed on interface ${interfaceName}`);
});
socket.bind({
address: addressInfo.address,
port: options.localPort,
exclusive: false, // Allow multiple sockets to bind to the same address and port
}, () => {
// Send the SSDP M-SEARCH message
socket.send(searchMessage, options.port, options.multicastIp, (err, bytes) => {
if (err) {
logger?.Error(`Error sending SSDP M-SEARCH on interface ${interfaceName}: ${err}`);
}
else {
logger?.Info(`Send SSDP M-SEARCH (${bytes}bytes) on interface ${interfaceName}`);
}
});
});
}
});
});
const getLocation = (message) => {
const lines = message.toString().split(Strings_CR_LF);
if (lines && lines.length > 0) {
for (const line of lines) {
const index = line.indexOf(":");
if (index >= 0) {
const name = line.substring(0, index);
if (IsEqual(name, "Location", true)) {
const value = line.substring(index + 1);
return value.trim();
}
}
}
}
return undefined;
};
while (true) {
try {
const message = messageList.pop();
if (message === undefined) {
// End the timer
const endTime = process.hrtime(startTime);
// Calculate the elapsed time in milliseconds
const elapsedTimeInMs = (endTime[0] * 1000 + endTime[1] / 1e6);
if (elapsedTimeInMs > options.timeoutMilliseconds) {
return;
}
else {
await Sleep(100);
}
continue;
}
const location = getLocation(message);
if (!location) {
logger?.Warn("Location not found");
continue;
}
if (locationList.includes(location)) {
return;
}
logger?.Info(`discover: ${location}`);
locationList.push(location);
const r = await getFromLocation(location);
if (!r) {
continue;
}
const { ip, data } = r;
if (options.checkIp(ip)) {
KavenCache.SetValue(cacheKey, location);
return ip;
}
const serviceList = GetStringBetween(data, "<service>", "</service>");
for (const service of serviceList) {
try {
const serviceType = GetStringBetween(service, "<serviceType>", "</serviceType>", 1)[0];
const controlURL = GetStringBetween(service, "<controlURL>", "</controlURL>", 1)[0];
if (serviceType && controlURL) {
const url = new URL(controlURL, location).toString();
logger?.Info(`url: ${url}, serviceType: ${serviceType}`);
const actionOptions = {
url,
serviceType,
action: "GetExternalIPAddress",
actionResultName: "NewExternalIPAddress",
logger,
};
const ip = await ExecuteSoapAction(actionOptions);
if (options.checkIp(ip)) {
logger?.Info(`Obtained IP successfully: ${url}, ${serviceType}`);
KavenCache.SetValue(cacheKey, actionOptions);
return ip;
}
}
}
catch (ex) {
logger?.Warn(ex);
}
}
}
catch (ex) {
logger?.Warn(ex);
}
}
}
finally {
socketList.forEach(p => p.close());
}
}