@superadnim/osint-mcp-server
Version:
Professional OSINT MCP Server for intelligence gathering with privacy protection
260 lines • 9 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DataSourceManager = exports.EmailLookupDataSource = exports.PhoneLookupDataSource = exports.PeopleSearchDataSource = exports.GoogleSearchDataSource = exports.BaseDataSource = void 0;
const axios_1 = __importDefault(require("axios"));
class BaseDataSource {
httpClient;
rateLimiter;
cache;
logger;
apiKey;
constructor(rateLimiter, cache, logger, apiKey) {
this.rateLimiter = rateLimiter;
this.cache = cache;
this.logger = logger;
this.apiKey = apiKey;
this.httpClient = axios_1.default.create({
timeout: 30000,
headers: {
'User-Agent': '@superadnim/osint-mcp-server/1.0.5',
'Accept': 'application/json',
},
});
}
initializeSource() {
this.setupRateLimiting();
this.setupResponseInterceptors();
}
setupRateLimiting() {
this.rateLimiter.createLimiter(this.id, this.rateLimit);
}
setupResponseInterceptors() {
this.httpClient.interceptors.response.use((response) => {
this.logger.debug(`API request successful: ${this.id}`, {
url: response.config.url,
status: response.status,
});
return response;
}, (error) => {
this.logger.error(`API request failed: ${this.id}`, {
url: error.config?.url,
status: error.response?.status,
message: error.message,
});
return Promise.reject(error);
});
}
async makeRequest(config, cacheKey, cacheTTL) {
// Check cache first
if (cacheKey && this.cache.has(cacheKey)) {
this.logger.debug(`Cache hit for ${this.id}:${cacheKey}`);
return this.cache.get(cacheKey);
}
// Make rate-limited request
const response = await this.rateLimiter.schedule(this.id, async () => {
return this.httpClient.request(config);
});
// Cache successful response
if (cacheKey && response.data) {
this.cache.set(cacheKey, response.data, cacheTTL);
}
return response.data;
}
generateCacheKey(method, params) {
return this.cache.generateKey(`${this.id}:${method}`, params);
}
isAvailable() {
return this.apiKey !== undefined || this.tier === 1;
}
async getUsageStats() {
return {
queued: await this.rateLimiter.getQueueSize(this.id),
running: await this.rateLimiter.getPendingCount(this.id),
};
}
}
exports.BaseDataSource = BaseDataSource;
class GoogleSearchDataSource extends BaseDataSource {
id = 'google_search';
name = 'Google Search';
type = 'domain';
tier = 1;
baseUrl = 'https://www.google.com';
rateLimit = {
requestsPerMinute: 10,
requestsPerHour: 100,
requestsPerDay: 1000,
concurrent: 2,
};
endpoints = [];
constructor(rateLimiter, cache, logger, apiKey) {
super(rateLimiter, cache, logger, apiKey);
this.initializeSource();
}
async search(query, options = {}) {
const cacheKey = this.generateCacheKey('search', { query, ...options });
// For demonstration - in real implementation, this would parse Google results
// This is just a mock implementation
return this.makeRequest({
method: 'GET',
url: '/search',
params: {
q: query,
...options,
},
}, cacheKey, 3600);
}
}
exports.GoogleSearchDataSource = GoogleSearchDataSource;
class PeopleSearchDataSource extends BaseDataSource {
id = 'fastpeoplesearch';
name = 'FastPeopleSearch';
type = 'people';
tier = 2;
baseUrl = 'https://www.fastpeoplesearch.com';
rateLimit = {
requestsPerMinute: 5,
requestsPerHour: 50,
requestsPerDay: 200,
concurrent: 1,
};
endpoints = [];
constructor(rateLimiter, cache, logger, apiKey) {
super(rateLimiter, cache, logger, apiKey);
this.initializeSource();
}
async search(query, options = {}) {
const cacheKey = this.generateCacheKey('search', { query, ...options });
// Mock implementation - would implement actual people search logic
return {
query,
results: [],
source: this.name,
timestamp: new Date().toISOString(),
};
}
}
exports.PeopleSearchDataSource = PeopleSearchDataSource;
class PhoneLookupDataSource extends BaseDataSource {
id = 'numverify';
name = 'NumVerify';
type = 'phone';
tier = 2;
baseUrl = 'http://apilayer.net/api';
rateLimit = {
requestsPerMinute: 10,
requestsPerHour: 250,
requestsPerDay: 1000,
concurrent: 3,
};
endpoints = [];
constructor(rateLimiter, cache, logger, apiKey) {
super(rateLimiter, cache, logger, apiKey);
this.initializeSource();
}
async search(query, options = {}) {
if (!this.apiKey) {
throw new Error('NumVerify API key required');
}
const cacheKey = this.generateCacheKey('validate', { number: query, ...options });
return this.makeRequest({
method: 'GET',
url: '/validate',
params: {
access_key: this.apiKey,
number: query,
...options,
},
}, cacheKey, 3600);
}
}
exports.PhoneLookupDataSource = PhoneLookupDataSource;
class EmailLookupDataSource extends BaseDataSource {
id = 'hunter_io';
name = 'Hunter.io';
type = 'email';
tier = 2;
baseUrl = 'https://api.hunter.io/v2';
rateLimit = {
requestsPerMinute: 10,
requestsPerHour: 100,
requestsPerDay: 1000,
concurrent: 2,
};
endpoints = [];
constructor(rateLimiter, cache, logger, apiKey) {
super(rateLimiter, cache, logger, apiKey);
this.initializeSource();
}
async search(query, options = {}) {
if (!this.apiKey) {
throw new Error('Hunter.io API key required');
}
const cacheKey = this.generateCacheKey('email-verifier', { email: query, ...options });
return this.makeRequest({
method: 'GET',
url: '/email-verifier',
params: {
api_key: this.apiKey,
email: query,
...options,
},
}, cacheKey, 3600);
}
}
exports.EmailLookupDataSource = EmailLookupDataSource;
class DataSourceManager {
sources = new Map();
logger;
constructor(rateLimiter, cache, logger, apiKeys) {
this.logger = logger;
// Initialize available data sources
this.sources.set('google_search', new GoogleSearchDataSource(rateLimiter, cache, logger));
this.sources.set('fastpeoplesearch', new PeopleSearchDataSource(rateLimiter, cache, logger));
this.sources.set('numverify', new PhoneLookupDataSource(rateLimiter, cache, logger, apiKeys.numverify));
this.sources.set('hunter_io', new EmailLookupDataSource(rateLimiter, cache, logger, apiKeys.hunter_io));
}
getSource(sourceId) {
return this.sources.get(sourceId);
}
getSourcesByType(type) {
return Array.from(this.sources.values()).filter(source => source.type === type);
}
getAvailableSources() {
return Array.from(this.sources.values()).filter(source => source.isAvailable());
}
async searchAcrossSources(type, query, options = {}) {
const sources = this.getSourcesByType(type).filter(source => source.isAvailable());
if (sources.length === 0) {
this.logger.warn(`No available sources for type: ${type}`);
return [];
}
const promises = sources.map(async (source) => {
try {
const result = await source.search(query, options);
return {
source: source.id,
data: result,
success: true,
};
}
catch (error) {
this.logger.error(`Search failed for source ${source.id}`, { error });
return {
source: source.id,
error: error instanceof Error ? error.message : String(error),
success: false,
};
}
});
const results = await Promise.allSettled(promises);
return results
.filter(result => result.status === 'fulfilled')
.map(result => result.value);
}
}
exports.DataSourceManager = DataSourceManager;
//# sourceMappingURL=data-sources.js.map