@qiniu/miku-delivery-mp-ks
Version:
Kuaishou Mini Program SDK for Miku Delivery
299 lines (298 loc) • 13.6 kB
JavaScript
/**
* @file DNS 查询组件
* @desc 详见 https://cf.qiniu.io/pages/viewpage.action?pageId=107318871
* @todo - dns 对外接口（`ResolveResult`）的调整（干掉 ecdn 特别的概念）
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
import { Polyfilled_URL } from '../utils/url';
import ConsistentHash from '../utils/consistent-hash';
import Context from '../utils/context';
import { getDebug } from '../utils/debug';
import { timeMinus, waitTimeout } from '../utils/index';
import { httpResolve, isECDNResolveResult, matchRule, shouldServeWithRule } from './httpdns';
import { DoWithNodeError, isAbortError, LocalDNSError, NoAvailableNodeError, select, ResolveError, shouldDisableTarget } from './common';
import { getCacheKey } from './cache-key';
import { collapseCalls } from '../utils/async';
import { getStorage, setStorage } from '../utils/storage';
const debug = getDebug('dns');
const storageNamespace = 'miku-delivery/dns/v1';
/** DNS 查询组件 */
export default class CDNResolver {
constructor(logger, app,
/** 客户端唯一标识 */
clientID, dnsResolver = httpResolve,
/** Resolve（`resolveDomain`）过程的超时时间 */
resolveTimeout = 2000,
/** elt 第一次失败禁用时长 */
initialDisableDuration = 5000) {
Object.defineProperty(this, "logger", {
enumerable: true,
configurable: true,
writable: true,
value: logger
});
Object.defineProperty(this, "app", {
enumerable: true,
configurable: true,
writable: true,
value: app
});
Object.defineProperty(this, "clientID", {
enumerable: true,
configurable: true,
writable: true,
value: clientID
});
Object.defineProperty(this, "dnsResolver", {
enumerable: true,
configurable: true,
writable: true,
value: dnsResolver
});
Object.defineProperty(this, "resolveTimeout", {
enumerable: true,
configurable: true,
writable: true,
value: resolveTimeout
});
Object.defineProperty(this, "initialDisableDuration", {
enumerable: true,
configurable: true,
writable: true,
value: initialDisableDuration
});
/** 缓存节点禁用信息 */
Object.defineProperty(this, "disabledElts", {
enumerable: true,
configurable: true,
writable: true,
value: new Map()
});
/** 从持久化存储中读取解析结果，注意这里做了合并调用的处理，以减少初始化时对持久化存储的读操作 */
Object.defineProperty(this, "getResolveResultFromStorage", {
enumerable: true,
configurable: true,
writable: true,
value: collapseCalls((domain) => __awaiter(this, void 0, void 0, function* () {
const fromStorage = yield getStorage(this.getStorageKey(domain));
return fromStorage;
}), d => d)
});
/** 更新持久化存储中的结果，注意这里做了合并调用的处理，以避免重复的发往远端 HTTPDNS 服务器的请求 */
Object.defineProperty(this, "refreshResolveResultInStorage", {
enumerable: true,
configurable: true,
writable: true,
value: collapseCalls((ctx, domain) => __awaiter(this, void 0, void 0, function* () {
const result = yield this.dnsResolver(ctx, domain, this.app, this.logger);
const expireAt = Date.now() + (result.ttl * 1000);
const resultToSave = Object.assign(Object.assign({}, result), { domain, expireAt });
this.saveResolveResultToStorage(domain, resultToSave);
return resultToSave;
}), (_, d) => d)
});
}
/** 判断节点是否被禁用 */
isEltDisabled(eltId) {
const disabledElt = this.disabledElts.get(eltId);
if (disabledElt == null)
return false;
return disabledElt.after > Date.now();
}
/** 节点失败时更新禁用信息，连续失败罚时翻倍 */
setEltDisabled(eltId) {
const disabledElt = this.disabledElts.get(eltId);
const duration = disabledElt == null ? this.initialDisableDuration : disabledElt.duration;
const after = Date.now() + duration;
this.disabledElts.set(eltId, { duration: duration * 2, after });
}
/** 节点成功时清除禁用信息 */
removeEltDisabled(eltId) {
if (this.disabledElts.has(eltId)) {
this.disabledElts.delete(eltId);
}
}
getCacheKey(url, domain) {
return __awaiter(this, void 0, void 0, function* () {
// 注意这里可能会影响首开性能（因为首开逻辑也在得到 cache key 之后，而如无本地（内存或 IndexedDB）缓存，cache key 的计算逻辑可能要等 HTTPDNS 接口调用完成）；
// 如确实评估影响较大，可以考虑优化为：如果本地有 HTTPDNS 结果的缓存，则使用之，否则直接走默认逻辑（同 non-ECDN domain 逻辑），而不去等待调用 HTTPDNS 接口完成
const resolveResult = yield this.resolveDomain(new Context(), domain);
if (!isECDNResolveResult(resolveResult))
return url;
return getCacheKey(url, resolveResult.cacheKey, resolveResult.bucket);
});
}
getStorageKey(domain) {
return `${storageNamespace}/${domain}`;
}
/** 向持久化存储中写入解析结果 */
saveResolveResultToStorage(domain, result) {
return __awaiter(this, void 0, void 0, function* () {
return setStorage(this.getStorageKey(domain), result);
});
}
/** 给定域名并获取解析结果 */
resolveDomain(ctx, domain) {
return __awaiter(this, void 0, void 0, function* () {
const startAt = Date.now();
const resolvePromise = (() => __awaiter(this, void 0, void 0, function* () {
const fromStorage = yield this.getResolveResultFromStorage(domain);
if (fromStorage == null)
return this.refreshResolveResultInStorage(ctx, domain);
if (isStale(fromStorage)) { // 若缓存过期，继续使用之，但触发异步的更新
debug('result from storage cache expired', domain);
this.refreshResolveResultInStorage(ctx, domain);
}
ctx.set('dnsResolveFromCache', true);
return fromStorage;
}))().then(r => {
ctx.set('dnsResolveTotalTime', timeMinus(Date.now(), startAt)); // 供后续上报 DoLog 使用
return r;
});
return Promise.race([
resolvePromise,
waitTimeout(this.resolveTimeout, 'Resolve domain')
]).catch(e => {
console.warn('DNS resolve failed:', e);
throw new ResolveError(e);
});
});
}
/** 给定 URL，解析并使用结果 IP / Host 执行特定逻辑（job） */
do(ctx, url, attempts, job) {
return __awaiter(this, void 0, void 0, function* () {
let err;
debug('Resolve for', url);
const domain = new Polyfilled_URL(url).hostname;
const [cacheKey, resolveResult] = yield Promise.all([
this.getCacheKey(url, domain),
this.resolveDomain(ctx, domain)
]);
if (!isECDNResolveResult(resolveResult))
throw new LocalDNSError(`Non-ECDN resolve result: ${domain}`);
const { groups: allGroups, rules } = resolveResult;
if (allGroups.length === 0)
throw new NoAvailableNodeError(`No available group for ${domain}`);
debug('Resolved for', url);
const rule = matchRule(rules, cacheKey);
if (rule == null)
throw new LocalDNSError('No matched rule found');
if (!shouldServeWithRule(rule))
throw new LocalDNSError('Do not serve with given rule');
let groups = rule.groups.map(({ groupidx, weight, splitn }) => {
const group = allGroups[groupidx];
if (group == null)
throw new Error(`Unexpected groupidx: ${groupidx}`);
return { group, weight, splitn };
});
for (let i = 0; i < attempts; i++) {
const group = select(groups, this.clientID, cacheKey, g => g.weight);
if (group == null)
break;
try {
debug('doWithGroup', i, cacheKey);
yield this.doWithGroup(ctx, group.group, cacheKey, group.splitn, job);
return;
}
catch (e) {
console.warn('doWithGroup (weight:', group.weight, ') failed:', e, cacheKey);
err = e;
if (isAbortError(e))
break;
if (shouldDisableTarget(e)) {
groups = groups.filter(g => g !== group);
}
}
}
err = isAbortError(err) ? err : new DoWithNodeError(err);
throw err;
});
}
doWithGroup(ctx, group, cacheKey, splitn, job) {
return __awaiter(this, void 0, void 0, function* () {
const availableElts = group.elts.filter(elt => !this.isEltDisabled(elt.id));
if (availableElts.length === 0)
throw new Error('No available elt');
const consistentHash = makeConsistentHash(group);
const availableMembers = new Set(availableElts.map(elt => elt.id));
const eltNum = splitn === 1 ? 2 : splitn; // 至少取两个候选节点出来，以确保在 splitn: 1 时第一次失败且需要排除时还有第二个节点可以重试
let elts = consistentHash.getN(cacheKey, eltNum, availableMembers).map(id => availableElts.find(e => e.id === id));
let err;
for (let i = 0; i < 2; i++) { // 尝试两次
const elt = select(elts.slice(0, splitn), this.clientID, cacheKey, e => e.replicas);
if (elt == null)
break;
try {
debug('doWithElt', elt.id, cacheKey);
ctx.set('downloadEltID', elt.id);
yield doWithElt(elt, this.clientID, cacheKey, job);
this.removeEltDisabled(elt.id);
return;
}
catch (e) {
console.warn('doWithElt', elt.id, 'failed:', e, cacheKey);
err = e;
if (isAbortError(e))
break;
if (shouldDisableTarget(e)) {
elts = elts.filter(e => e !== elt);
this.setEltDisabled(elt.id);
}
}
}
throw err;
});
}
}
// 构造 consistent hash 成本相对高（主要是 set 时的 hash 计算），这里做一个简单的缓存
const consistentHashCache = new Map();
export function makeConsistentHash({ elts }) {
const cacheKey = elts.map(({ id, replicas }) => `${id}:${replicas}`).join(',');
const fromCache = consistentHashCache.get(cacheKey);
if (fromCache != null)
return fromCache;
const consistentHash = new ConsistentHash();
consistentHash.set(elts.map(e => ({ key: e.id, replicas: e.replicas })));
consistentHashCache.set(cacheKey, consistentHash);
return consistentHash;
}
export function doWithElt(elt, clientID, cacheKey, job) {
return __awaiter(this, void 0, void 0, function* () {
let addrs = elt.addrs;
if (addrs.length === 0)
throw new Error(`Empty addr list`);
let err;
for (let i = 0; i < 2; i++) {
const curr = select(addrs, clientID, cacheKey);
if (curr == null)
break;
try {
debug('do job with', curr, cacheKey);
yield job(curr);
return;
}
catch (e) {
console.warn('do job with', curr, 'failed:', e, cacheKey);
err = e;
if (isAbortError(e))
break;
if (shouldDisableTarget(e)) {
addrs = addrs.filter(i => i !== curr);
}
}
}
throw err;
});
}
/** 是否结果已经过期 */
function isStale(result) {
return result.expireAt < Date.now();
}