@mikkelscheike/email-provider-links
Version:
TypeScript library for email provider detection with 140 providers (259 domains), concurrent DNS resolution, alias normalization, and HTTPS login URL validation for login and password reset flows
531 lines • 19.9 kB
JavaScript
"use strict";
/**
* Concurrent DNS Detection Engine
*
* Implements parallel MX/TXT record lookups for 2x faster business domain detection.
* Uses Promise.allSettled for fault tolerance and intelligent result merging.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.ConcurrentDNSDetector = void 0;
exports.hostnameMatchesPattern = hostnameMatchesPattern;
exports.resetDnsRateLimiter = resetDnsRateLimiter;
exports.setDnsRateLimit = setDnsRateLimit;
exports.createConcurrentDNSDetector = createConcurrentDNSDetector;
exports.clearDnsResultCache = clearDnsResultCache;
exports.detectProviderConcurrent = detectProviderConcurrent;
const util_1 = require("util");
const dns_1 = require("dns");
const constants_1 = require("./constants");
// Convert Node.js callback-style DNS functions to Promise-based
const resolveMxAsync = (0, util_1.promisify)(dns_1.resolveMx);
const resolveTxtAsync = (0, util_1.promisify)(dns_1.resolveTxt);
/**
* True when hostname equals pattern or is a DNS subdomain of pattern.
* Avoids substring false positives (e.g. "notgoogle.com" vs "google.com").
*/
function hostnameMatchesPattern(hostname, pattern) {
const host = hostname.toLowerCase().replace(/\.+$/, '');
const pat = pattern.toLowerCase().replace(/\.+$/, '');
if (!host || !pat)
return false;
return host === pat || host.endsWith('.' + pat);
}
/**
* Process-wide sliding-window rate limiter for DNS lookups.
*/
class DnsRateLimiter {
timestamps = [];
maxPerMinute;
constructor(maxPerMinute = constants_1.DnsConstants.MAX_REQUESTS_PER_MINUTE) {
this.maxPerMinute = maxPerMinute;
}
setLimit(maxPerMinute) {
this.maxPerMinute = maxPerMinute;
}
reset() {
this.timestamps = [];
}
/**
* Records a DNS detection attempt. Throws if the limit is exceeded.
* Skipped in test environments unless FORCE_DNS_RATE_LIMIT=1.
*/
acquire() {
const forceInTests = process.env.FORCE_DNS_RATE_LIMIT === '1';
if (!forceInTests && (process.env.NODE_ENV === 'test' || process.env.JEST_WORKER_ID)) {
return;
}
const now = Date.now();
const windowMs = 60_000;
this.timestamps = this.timestamps.filter(t => now - t < windowMs);
if (this.timestamps.length >= this.maxPerMinute) {
const oldest = this.timestamps[0] ?? now;
const retryAfterMs = windowMs - (now - oldest);
const retryAfterSec = Math.max(1, Math.ceil(retryAfterMs / 1000));
throw new Error(`Rate limit exceeded. Try again in ${retryAfterSec} seconds`);
}
this.timestamps.push(now);
}
}
const dnsRateLimiter = new DnsRateLimiter();
/** Reset rate limiter state (for tests). */
function resetDnsRateLimiter() {
dnsRateLimiter.reset();
clearDnsResultCache();
}
/** Override the per-minute limit (for tests / advanced config). */
function setDnsRateLimit(maxPerMinute) {
dnsRateLimiter.setLimit(maxPerMinute);
}
/**
* Default configuration for concurrent DNS detection
*/
const DEFAULT_CONFIG = {
timeout: 5000,
enableParallel: true,
prioritizeMX: true,
collectDebugInfo: false,
fallbackToSequential: true
};
/**
* Concurrent DNS Detection Engine
*/
class ConcurrentDNSDetector {
// Store active query states
activeQueries = new Set();
// Cleanup method for tests
cleanup() {
// Cancel any in-progress timeouts
const timeoutError = new Error('Operation cancelled by cleanup');
for (const { reject } of this.activeQueries) {
reject(timeoutError);
}
this.activeQueries.clear();
return Promise.resolve();
}
config;
providers;
constructor(providers, config = {}) {
this.config = { ...DEFAULT_CONFIG, ...config };
this.providers = providers.filter(p => p.customDomainDetection &&
(p.customDomainDetection.mxPatterns || p.customDomainDetection.txtPatterns));
}
/**
* Detect provider for a domain using concurrent DNS lookups
*/
async detectProvider(domain) {
// Enforce process-wide DNS rate limit before any network I/O
dnsRateLimiter.acquire();
const startTime = Date.now();
const normalizedDomain = domain.toLowerCase().trim().replace(/\.+$/, '');
// Initialize result
const result = {
provider: null,
detectionMethod: null,
confidence: 0,
timing: { mx: 0, txt: 0, total: 0 },
debug: this.config.collectDebugInfo ? {
mxMatches: [],
txtMatches: [],
conflicts: false,
queries: [],
fallbackUsed: false
} : undefined
};
try {
let queries;
if (this.config.enableParallel) {
queries = await this.performParallelQueries(normalizedDomain);
}
else {
queries = await this.performSequentialQueries(normalizedDomain);
}
// Update timing information
result.timing = this.calculateTiming(queries, startTime);
if (this.config.collectDebugInfo && result.debug) {
result.debug.queries = queries;
}
// Find provider matches
const matches = this.findProviderMatches(queries);
if (this.config.collectDebugInfo && result.debug) {
result.debug.mxMatches = matches.filter(m => m.method === 'mx_record').map(m => m.provider.companyProvider);
result.debug.txtMatches = matches.filter(m => m.method === 'txt_record').map(m => m.provider.companyProvider);
result.debug.conflicts = matches.length > 1;
}
// Select best match
const bestMatch = this.selectBestMatch(matches);
if (bestMatch) {
result.provider = bestMatch.provider;
result.detectionMethod = bestMatch.method;
result.confidence = bestMatch.confidence;
}
else {
// Check for proxy services
const proxyResult = this.detectProxy(queries);
if (proxyResult) {
result.detectionMethod = 'proxy_detected';
result.proxyService = proxyResult;
result.confidence = 0.9; // High confidence in proxy detection
}
}
}
catch (error) {
// Handle fallback to sequential if parallel fails
if (this.config.enableParallel && this.config.fallbackToSequential) {
if (this.config.collectDebugInfo && result.debug) {
result.debug.fallbackUsed = true;
}
try {
const fallbackQueries = await this.performSequentialQueries(normalizedDomain);
result.timing = this.calculateTiming(fallbackQueries, startTime);
const matches = this.findProviderMatches(fallbackQueries);
const bestMatch = this.selectBestMatch(matches);
if (bestMatch) {
result.provider = bestMatch.provider;
result.detectionMethod = bestMatch.method;
result.confidence = bestMatch.confidence * 0.9; // Slightly lower confidence for fallback
}
}
catch (fallbackError) {
// Both parallel and sequential failed
console.warn('DNS detection failed:', fallbackError);
}
}
}
result.timing.total = Date.now() - startTime;
return result;
}
/**
* Perform DNS queries in parallel using Promise.allSettled with smart optimization
*/
async performParallelQueries(domain) {
const queries = [
this.queryMX(domain),
this.queryTXT(domain)
];
const results = await Promise.allSettled(queries);
const mappedResults = results.map((result, index) => {
if (result.status === 'fulfilled') {
return result.value;
}
else {
return {
type: index === 0 ? 'mx' : 'txt',
success: false,
error: result.reason,
timing: this.config.timeout
};
}
});
// If MX query succeeded and found a strong match, we can be confident
// and potentially ignore TXT timing for performance reporting
const mxResult = mappedResults[0];
const txtResult = mappedResults[1];
if (mxResult && mxResult.success && this.hasMXMatch(mxResult) && this.config.prioritizeMX) {
// Create an optimized TXT result that indicates it wasn't needed
const optimizedTxtResult = {
type: 'txt',
success: txtResult?.success || false,
records: txtResult?.records || [],
timing: 0 // Don't count TXT time if MX was sufficient
};
if (txtResult?.error) {
optimizedTxtResult.error = txtResult.error;
}
if (txtResult?.rawResponse) {
optimizedTxtResult.rawResponse = txtResult.rawResponse;
}
return [mxResult, optimizedTxtResult];
}
return mappedResults;
}
/**
* Perform DNS queries sequentially (fallback mode)
*/
async performSequentialQueries(domain) {
const results = [];
// Try MX first
try {
const mxResult = await this.queryMX(domain);
results.push(mxResult);
// If MX succeeds and finds a match, we might skip TXT for performance
if (mxResult.success && this.hasMXMatch(mxResult)) {
// Add a placeholder TXT result
results.push({
type: 'txt',
success: false,
timing: 0,
error: new Error('Skipped due to MX match')
});
return results;
}
}
catch (error) {
results.push({
type: 'mx',
success: false,
error: error,
timing: this.config.timeout
});
}
// Try TXT
try {
const txtResult = await this.queryTXT(domain);
results.push(txtResult);
}
catch (error) {
results.push({
type: 'txt',
success: false,
error: error,
timing: this.config.timeout
});
}
return results;
}
/**
* Query MX records with timeout
*/
async queryMX(domain) {
const startTime = Date.now();
try {
const records = await this.withTimeout(() => resolveMxAsync(domain), this.config.timeout);
return {
type: 'mx',
success: true,
records,
timing: Date.now() - startTime,
rawResponse: this.config.collectDebugInfo ? records : undefined
};
}
catch (error) {
return {
type: 'mx',
success: false,
error: error,
timing: Date.now() - startTime
};
}
}
/**
* Query TXT records with timeout
*/
async queryTXT(domain) {
const startTime = Date.now();
try {
const records = await this.withTimeout(() => resolveTxtAsync(domain), this.config.timeout);
const flatRecords = records.flat();
return {
type: 'txt',
success: true,
records: flatRecords,
timing: Date.now() - startTime,
rawResponse: this.config.collectDebugInfo ? records : undefined
};
}
catch (error) {
return {
type: 'txt',
success: false,
error: error,
timing: Date.now() - startTime
};
}
}
/**
* Find provider matches from DNS query results
*/
findProviderMatches(queries) {
const matches = [];
for (const query of queries) {
if (!query.success || !query.records)
continue;
for (const provider of this.providers) {
const match = this.matchProvider(provider, query);
if (match) {
matches.push(match);
}
}
}
return matches;
}
/**
* Match a provider against DNS query results
*/
matchProvider(provider, query) {
if (!provider.customDomainDetection || !query.records)
return null;
const detection = provider.customDomainDetection;
let matchedPatterns = [];
let confidence = 0;
if (query.type === 'mx' && detection.mxPatterns) {
for (const record of query.records) {
if (typeof record !== 'object' || record === null)
continue;
const exchange = record.exchange?.toLowerCase() || '';
for (const pattern of detection.mxPatterns) {
if (hostnameMatchesPattern(exchange, pattern)) {
matchedPatterns.push(pattern);
confidence = Math.max(confidence, 0.9); // High confidence for MX matches
}
}
}
}
else if (query.type === 'txt' && detection.txtPatterns) {
for (const record of query.records) {
if (typeof record !== 'string')
continue;
const txtRecord = record.toLowerCase();
for (const pattern of detection.txtPatterns) {
if (txtRecord.includes(pattern.toLowerCase())) {
matchedPatterns.push(pattern);
confidence = Math.max(confidence, 0.7); // Medium confidence for TXT matches
}
}
}
}
if (matchedPatterns.length > 0) {
return {
provider,
method: query.type === 'mx' ? 'mx_record' : 'txt_record',
confidence: confidence * (matchedPatterns.length / (detection.mxPatterns?.length || detection.txtPatterns?.length || 1)),
matchedPatterns
};
}
return null;
}
/**
* Select the best provider match from multiple candidates
*/
selectBestMatch(matches) {
if (matches.length === 0)
return null;
if (matches.length === 1)
return matches[0] ?? null;
// Sort by confidence and preference for MX records
const sortedMatches = matches.sort((a, b) => {
// Prioritize MX records if configured
if (this.config.prioritizeMX) {
if (a.method === 'mx_record' && b.method !== 'mx_record')
return -1;
if (b.method === 'mx_record' && a.method !== 'mx_record')
return 1;
}
// Then by confidence
return b.confidence - a.confidence;
});
return sortedMatches.length > 0 ? (sortedMatches[0] ?? null) : null;
}
/**
* Check if MX result has potential matches (for sequential optimization)
*/
hasMXMatch(mxResult) {
if (!mxResult.success || !mxResult.records)
return false;
for (const provider of this.providers) {
const match = this.matchProvider(provider, mxResult);
if (match)
return true;
}
return false;
}
/**
* Detect proxy services from DNS results
*/
detectProxy(queries) {
const mxQuery = queries.find(q => q.type === 'mx' && q.success);
if (!mxQuery?.records)
return null;
for (const record of mxQuery.records) {
if (typeof record !== 'object' || record === null)
continue;
const exchange = record.exchange?.toLowerCase() || '';
for (const provider of this.providers) {
if (provider.type === 'proxy_service' && provider.customDomainDetection?.mxPatterns) {
for (const pattern of provider.customDomainDetection.mxPatterns) {
if (hostnameMatchesPattern(exchange, pattern)) {
return provider.companyProvider;
}
}
}
}
}
return null;
}
/**
* Calculate timing information from query results
*/
calculateTiming(queries, startTime) {
const mxQuery = queries.find(q => q.type === 'mx');
const txtQuery = queries.find(q => q.type === 'txt');
return {
mx: mxQuery?.timing || 0,
txt: txtQuery?.timing || 0,
total: Date.now() - startTime
};
}
/**
* Wrap a promise with a timeout
*/
withTimeout(fn, ms) {
// For extremely small timeouts, avoid starting the underlying DNS query at all
if (ms <= 1) {
return Promise.reject(new Error(`DNS query timeout after ${ms}ms`));
}
let rejectFn;
let queryEntry;
const wrappedPromise = new Promise((resolve, reject) => {
rejectFn = reject;
const timeout = setTimeout(() => reject(new Error(`DNS query timeout after ${ms}ms`)), ms);
timeout.unref?.();
// Start the underlying operation only after setting up the timeout
fn()
.then(resolve)
.catch(reject)
.finally(() => {
clearTimeout(timeout);
// Clean up active query
if (queryEntry) {
this.activeQueries.delete(queryEntry);
}
});
});
// Track active query for potential cleanup in tests
if (rejectFn) {
queryEntry = { promise: wrappedPromise, reject: rejectFn };
this.activeQueries.add(queryEntry);
}
return wrappedPromise;
}
}
exports.ConcurrentDNSDetector = ConcurrentDNSDetector;
/**
* Factory function to create a concurrent DNS detector
*/
function createConcurrentDNSDetector(providers, config) {
return new ConcurrentDNSDetector(providers, config);
}
/**
* Utility function for quick concurrent DNS detection.
* Results are cached per domain for a short TTL to avoid repeated lookups.
*/
const DNS_RESULT_CACHE_TTL_MS = 5 * 60 * 1000;
const dnsResultCache = new Map();
function clearDnsResultCache() {
dnsResultCache.clear();
}
async function detectProviderConcurrent(domain, providers, config) {
const normalizedDomain = domain.toLowerCase().trim().replace(/\.+$/, '');
const cacheKey = `${normalizedDomain}|${config?.timeout ?? DEFAULT_CONFIG.timeout}|${config?.enableParallel !== false}|${!!config?.collectDebugInfo}`;
const cached = dnsResultCache.get(cacheKey);
if (cached && cached.expires > Date.now()) {
return cached.result;
}
const detector = createConcurrentDNSDetector(providers, config);
const result = await detector.detectProvider(normalizedDomain);
dnsResultCache.set(cacheKey, {
expires: Date.now() + DNS_RESULT_CACHE_TTL_MS,
result
});
return result;
}
//# sourceMappingURL=concurrent-dns.js.map