endpoint-sentinel
Version:
User-friendly security scanner with interactive setup that scales from beginner to expert
581 lines • 23.1 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.PracticalDiscoveryEngine = void 0;
const crypto = __importStar(require("crypto"));
class PracticalDiscoveryEngine {
httpClient;
rateLimiter;
logger;
visitedUrls = new Set();
seenStates = [];
contentHashes = new Map(); // FOR SPA DETECTION
// Enhanced regex patterns for better route extraction
patterns = {
// Enhanced SPA route patterns - MUCH BETTER
spaRoutes: [
/"\/[a-zA-Z0-9_-]+(?:\/[a-zA-Z0-9_-]+)*"/g, // "/route" or "/route/subroute"
/'\/[a-zA-Z0-9_-]+(?:\/[a-zA-Z0-9_-]+)*'/g, // '/route' or '/route/subroute'
/path:\s*["']\/[a-zA-Z0-9_-]+(?:\/[a-zA-Z0-9_-]+)*["']/g, // React Router paths
/to:\s*["']\/[a-zA-Z0-9_-]+(?:\/[a-zA-Z0-9_-]+)*["']/g, // Navigation 'to' props
/route:\s*["']\/[a-zA-Z0-9_-]+(?:\/[a-zA-Z0-9_-]+)*["']/g, // Route definitions
],
// API endpoints from JavaScript
apiEndpoints: [
/['"](\/api\/[^'"]*)['"]/gi,
/fetch\(['"`]([^'"`]+)['"`]\)/gi,
/axios\.[get|post|put|delete|patch]+\(['"`]([^'"`]+)['"`]\)/gi,
/\$\.(get|post|put|delete|ajax)\(['"`]([^'"`]+)['"`]\)/gi,
],
// Form actions and endpoints
forms: [
/<form[^>]+action=['"]([^'"]+)['"]/gi,
/<input[^>]+formaction=['"]([^'"]+)['"]/gi,
],
// Links and resources
links: [
/<a[^>]+href=['"]([^'"]+)['"]/gi,
/<link[^>]+href=['"]([^'"]+)['"]/gi,
/<script[^>]+src=['"]([^'"]+)['"]/gi,
/<img[^>]+src=['"]([^'"]+)['"]/gi,
/<iframe[^>]+src=['"]([^'"]+)['"]/gi,
],
// CSS url() references
cssUrls: [
/url\(['"`]?([^'"`\)]+)['"`]?\)/gi,
]
};
constructor(httpClient, rateLimiter, logger) {
this.httpClient = httpClient;
this.rateLimiter = rateLimiter;
this.logger = logger;
}
async discoverEndpoints(startUrl, config) {
this.logger.info('Starting ENHANCED endpoint discovery', { startUrl, keywords: config.keywords });
const discovered = [];
const baseUrl = new URL(startUrl);
// PHASE 1: Check if this is a SPA
const isSPA = await this.detectSPA(startUrl);
this.logger.info(`SPA Detection: ${isSPA ? 'YES' : 'NO'}`, { target: startUrl });
if (isSPA) {
// SPA: Focus on JavaScript analysis
this.logger.info('Analyzing SPA JavaScript bundles...');
discovered.push(...await this.analyzeSPARoutes(startUrl, config));
}
else {
// Traditional web app: Use standard crawling
this.logger.info('Using traditional web crawling...');
discovered.push(...await this.traditionalCrawl(startUrl, config));
}
// PHASE 2: Add keyword-based intelligent guessing (only as fallback)
if (config.keywords && config.keywords.length > 0) {
this.logger.info('Adding keyword-based endpoint suggestions...');
discovered.push(...await this.addKeywordEndpoints(baseUrl, config.keywords));
}
// PHASE 3: Basic infrastructure endpoints (robots.txt, sitemap.xml)
discovered.push(...await this.addInfrastructureEndpoints(baseUrl));
const uniqueEndpoints = this.deduplicateEndpoints(discovered);
this.logger.info(`Discovery complete: ${uniqueEndpoints.length} endpoints found`, {
spa: discovered.filter(ep => ep.source === 'javascript_routes').length,
javascript: discovered.filter(ep => ep.source === 'api_discovery').length,
html: discovered.filter(ep => ep.source === 'html_links').length,
keywords: discovered.filter(ep => ep.source === 'wordlist').length
});
return uniqueEndpoints;
}
/**
* Detect if the target is a Single Page Application
*/
async detectSPA(startUrl) {
const baseUrl = new URL(startUrl);
const testPaths = ['/', '/nonexistent-test-path-12345'];
const contentHashes = [];
for (const path of testPaths) {
try {
const testUrl = new URL(path, baseUrl).href;
const response = await this.fetchWithRateLimit(testUrl);
if (response && response.status === 200 && response.data) {
const hash = crypto.createHash('md5').update(String(response.data)).digest('hex');
contentHashes.push(hash);
this.contentHashes.set(testUrl, hash);
}
}
catch (error) {
// Ignore errors for SPA detection
}
}
// If both paths return identical content, it's likely a SPA
const isSPA = contentHashes.length === 2 && contentHashes[0] === contentHashes[1];
return isSPA;
}
/**
* Analyze SPA for real routes in JavaScript bundles
*/
async analyzeSPARoutes(startUrl, _config) {
const endpoints = [];
const baseUrl = new URL(startUrl);
try {
// Get the main page to find JS bundles
const response = await this.fetchWithRateLimit(startUrl);
if (!response)
return endpoints;
// Extract JavaScript file references
const jsFiles = this.extractJavaScriptFiles(response.data, baseUrl);
this.logger.info(`Found ${jsFiles.length} JavaScript files to analyze`);
// Analyze each JavaScript file for routes
for (const jsUrl of jsFiles.slice(0, 3)) { // Limit to first 3 for performance
this.logger.info(`Analyzing JS bundle: ${jsUrl}`);
const jsResponse = await this.fetchWithRateLimit(jsUrl);
if (jsResponse && jsResponse.status === 200) {
const routes = this.extractSPARoutes(jsResponse.data, baseUrl);
endpoints.push(...routes);
this.logger.info(`Extracted ${routes.length} routes from ${jsUrl}`);
}
}
}
catch (error) {
this.logger.warn(`SPA analysis failed: ${error}`);
}
return endpoints;
}
/**
* Extract JavaScript files from HTML
*/
extractJavaScriptFiles(html, baseUrl) {
const jsFiles = [];
const scriptRegex = /<script[^>]+src=['"]([^'"]+)['"][^>]*>/gi;
let match;
while ((match = scriptRegex.exec(html)) !== null) {
try {
if (match[1]) {
const jsUrl = new URL(match[1], baseUrl).href;
if (jsUrl.endsWith('.js')) {
jsFiles.push(jsUrl);
}
}
}
catch (e) {
// Skip invalid URLs
}
}
return jsFiles;
}
/**
* Extract SPA routes from JavaScript content
*/
extractSPARoutes(jsContent, baseUrl) {
const routes = [];
const foundRoutes = new Set();
// Use enhanced patterns to find routes
this.patterns.spaRoutes.forEach(pattern => {
const matches = jsContent.matchAll(pattern);
for (const match of matches) {
let route = match[0].replace(/['"]/g, ''); // Remove quotes
// Clean up path: prefixes from React Router patterns
route = route.replace(/^(path|to|route):\s*/, '');
// Only include routes that look like real paths
if (this.isValidSPARoute(route)) {
foundRoutes.add(route);
}
}
});
// Convert routes to endpoints
foundRoutes.forEach(route => {
try {
const fullUrl = new URL(route, baseUrl).href;
routes.push({
url: fullUrl,
source: 'javascript_routes',
method: 'GET',
depth: 1,
confidence: 'high'
});
}
catch (e) {
// Skip invalid URLs
}
});
return routes;
}
/**
* Validate if a route looks like a real SPA route
*/
isValidSPARoute(route) {
// Must start with /
if (!route.startsWith('/'))
return false;
// Skip very short routes or just /
if (route.length <= 1)
return false;
// Skip routes with query params or fragments for now
if (route.includes('?') || route.includes('#'))
return false;
// Skip routes with file extensions (static files)
if (/\.[a-z]+$/i.test(route))
return false;
// Skip very generic patterns
if (/^\/[a-z]$/.test(route))
return false;
// Must contain valid characters
if (!/^\/[a-zA-Z0-9_\-\/]+$/.test(route))
return false;
return true;
}
/**
* Traditional crawling for non-SPA sites
*/
async traditionalCrawl(startUrl, config) {
const discovered = [];
const queue = [
{ url: startUrl, depth: 0 }
];
while (queue.length > 0 && discovered.length < 50) { // Limit for performance
const batch = queue.splice(0, config.maxConcurrency);
const results = await Promise.allSettled(batch.map(item => this.crawlUrl(item.url, item.depth, item.parent, config)));
for (let i = 0; i < results.length; i++) {
const result = results[i];
const item = batch[i];
if (result && item && result.status === 'fulfilled' && result.value) {
const endpoints = result.value;
discovered.push(...endpoints);
// Add new URLs to queue if within depth limit
if (item.depth < config.maxDepth) {
const newUrls = endpoints
.filter((ep) => !this.visitedUrls.has(ep.url))
.map((ep) => ({
url: ep.url,
depth: item.depth + 1,
parent: item.url
}));
queue.push(...newUrls);
}
}
}
}
return discovered;
}
/**
* Add keyword-based endpoints intelligently
*/
async addKeywordEndpoints(baseUrl, keywords) {
const endpoints = [];
// Only add keyword endpoints that make sense
const intelligentPatterns = [
'/{keyword}',
'/api/{keyword}',
'/{keyword}/dashboard',
'/{keyword}-practice',
'/{keyword}-assessment'
];
for (const keyword of keywords) {
for (const pattern of intelligentPatterns) {
const path = pattern.replace('{keyword}', keyword);
try {
endpoints.push({
url: new URL(path, baseUrl).href,
source: 'wordlist',
method: 'GET',
depth: 0,
confidence: 'low'
});
}
catch (e) {
// Skip invalid URLs
}
}
}
return endpoints;
}
/**
* Add basic infrastructure endpoints
*/
async addInfrastructureEndpoints(baseUrl) {
const endpoints = [];
const infraPaths = ['/robots.txt', '/sitemap.xml'];
for (const path of infraPaths) {
endpoints.push({
url: new URL(path, baseUrl).href,
source: 'robots_txt',
method: 'GET',
depth: 0,
confidence: 'medium'
});
}
return endpoints;
}
/**
* Fetch URL with rate limiting
*/
async fetchWithRateLimit(url) {
try {
await this.rateLimiter.throttle({
url,
method: 'GET',
headers: { 'User-Agent': this.getRandomUserAgent() },
timeout: 30000
});
return await this.httpClient.request({
url,
method: 'GET',
headers: {
'User-Agent': this.getRandomUserAgent(),
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
},
timeout: 30000
});
}
catch (error) {
this.logger.debug(`Failed to fetch ${url}:`, { error: String(error) });
return null;
}
}
async crawlUrl(url, depth, parent, config) {
if (this.visitedUrls.has(url)) {
return [];
}
this.visitedUrls.add(url);
try {
// Add realistic delay
if (config?.delays) {
const delay = Math.random() * (config.delays.max - config.delays.min) + config.delays.min;
await this.sleep(delay);
}
const response = await this.fetchWithRateLimit(url);
if (!response || response.status < 200 || response.status >= 400) {
return [];
}
return this.extractEndpointsFromResponse(url, response.data, depth, parent);
}
catch (error) {
this.logger.debug(`Failed to crawl ${url}:`, { error: String(error) });
return [];
}
}
extractEndpointsFromResponse(baseUrl, content, depth, parent) {
const endpoints = [];
// Extract from HTML
endpoints.push(...this.extractFromHTML(baseUrl, content, depth, parent));
// Extract JavaScript if content contains script tags or is JS file
if (content.includes('<script') || baseUrl.includes('.js')) {
endpoints.push(...this.extractFromJavaScript(baseUrl, content, depth, parent));
}
// Extract from CSS if content contains styles or is CSS file
if (content.includes('<style') || content.includes('url(') || baseUrl.includes('.css')) {
endpoints.push(...this.extractFromCSS(baseUrl, content, depth, parent));
}
// Check if this is a new state to avoid infinite loops
const currentUrls = endpoints.map(ep => ep.url);
if (this.isDuplicateState(currentUrls)) {
this.logger.debug(`Skipping duplicate state for ${baseUrl}`);
return [];
}
this.seenStates.push(currentUrls);
return endpoints;
}
extractFromHTML(baseUrl, html, depth, parent) {
const endpoints = [];
// Extract links
this.patterns.links.forEach(pattern => {
const matches = html.matchAll(pattern);
for (const match of matches) {
if (match[1]) {
const url = this.resolveUrl(baseUrl, match[1]);
if (url && this.isValidEndpoint(url)) {
const endpoint = {
url,
source: 'html_links',
method: 'GET',
depth,
confidence: 'medium'
};
if (parent) {
endpoint.parent = parent;
}
endpoints.push(endpoint);
}
}
}
});
// Extract forms
this.patterns.forms.forEach(pattern => {
const matches = html.matchAll(pattern);
for (const match of matches) {
if (match[1]) {
const url = this.resolveUrl(baseUrl, match[1]);
if (url && this.isValidEndpoint(url)) {
const endpoint = {
url,
source: 'html_links',
method: 'POST',
depth,
confidence: 'high'
};
if (parent) {
endpoint.parent = parent;
}
endpoints.push(endpoint);
}
}
}
});
return endpoints;
}
extractFromJavaScript(baseUrl, js, depth, parent) {
const endpoints = [];
this.patterns.apiEndpoints.forEach(pattern => {
const matches = js.matchAll(pattern);
for (const match of matches) {
const matchedUrl = match[1] || match[2];
if (matchedUrl) {
const url = this.resolveUrl(baseUrl, matchedUrl);
if (url && this.isValidEndpoint(url)) {
const method = this.inferMethodFromJS(match[0] || '');
const endpoint = {
url,
source: 'api_discovery',
method,
depth,
confidence: 'high'
};
if (parent) {
endpoint.parent = parent;
}
endpoints.push(endpoint);
}
}
}
});
return endpoints;
}
extractFromCSS(baseUrl, css, depth, parent) {
const endpoints = [];
this.patterns.cssUrls.forEach(pattern => {
const matches = css.matchAll(pattern);
for (const match of matches) {
if (match[1]) {
const url = this.resolveUrl(baseUrl, match[1]);
if (url && this.isValidEndpoint(url) && !this.isImageOrFont(url)) {
const endpoint = {
url,
source: 'html_links',
method: 'GET',
depth,
confidence: 'low'
};
if (parent) {
endpoint.parent = parent;
}
endpoints.push(endpoint);
}
}
}
});
return endpoints;
}
isDuplicateState(currentUrls) {
const currentSet = new Set(currentUrls);
return this.seenStates.some(stateUrls => {
const stateSet = new Set(stateUrls);
const intersection = new Set([...currentSet].filter(x => stateSet.has(x)));
const union = new Set([...currentSet, ...stateUrls]);
// Jaccard similarity > 0.8 indicates duplicate state
return intersection.size / union.size > 0.8;
});
}
inferMethodFromJS(matchText) {
const lowerMatch = matchText.toLowerCase();
if (lowerMatch.includes('.post') || lowerMatch.includes('"post"'))
return 'POST';
if (lowerMatch.includes('.put') || lowerMatch.includes('"put"'))
return 'PUT';
if (lowerMatch.includes('.delete') || lowerMatch.includes('"delete"'))
return 'DELETE';
if (lowerMatch.includes('.patch') || lowerMatch.includes('"patch"'))
return 'PATCH';
return 'GET';
}
isValidEndpoint(url) {
if (!url || url.length === 0)
return false;
if (url.startsWith('javascript:') || url.startsWith('mailto:'))
return false;
if (url.startsWith('#'))
return false; // Skip anchors
if (url.includes('logout') || url.includes('signout'))
return false; // Skip logout
// Must be HTTP/HTTPS or relative
return /^https?:\/\//i.test(url) || url.startsWith('/');
}
isImageOrFont(url) {
return /\.(jpg|jpeg|png|gif|svg|ico|woff|woff2|ttf|eot)$/i.test(url);
}
resolveUrl(baseUrl, relativeUrl) {
try {
if (relativeUrl.startsWith('http')) {
return relativeUrl;
}
const base = new URL(baseUrl);
return new URL(relativeUrl, base).toString();
}
catch {
return null;
}
}
deduplicateEndpoints(endpoints) {
const seen = new Set();
return endpoints.filter(ep => {
const key = `${ep.method}:${ep.url}`;
if (seen.has(key))
return false;
seen.add(key);
return true;
});
}
getRandomUserAgent() {
const userAgents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
];
const index = Math.floor(Math.random() * userAgents.length);
return userAgents[index];
}
sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
}
exports.PracticalDiscoveryEngine = PracticalDiscoveryEngine;
//# sourceMappingURL=discovery-engine.js.map