sec-edgar-toolkit
Version:
Open source toolkit to facilitate working with the SEC EDGAR database
144 lines • 5.73 kB
JavaScript
;
/**
* HTTP client for SEC EDGAR API with rate limiting and retry logic
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HttpClient = void 0;
const node_fetch_1 = __importDefault(require("node-fetch"));
const exceptions_1 = require("../exceptions");
const errors_1 = require("../exceptions/errors");
const cache_1 = require("./cache");
class HttpClient {
constructor(userAgent, options = {}) {
this.lastRequestTime = 0;
this.userAgent = userAgent;
this.rateLimitDelay = (options.rateLimitDelay || 0.1) * 1000; // Convert to milliseconds
this.maxRetries = options.maxRetries || 3;
this.timeout = options.timeout || 30000;
// Initialize cache if not explicitly disabled
if (options.cache !== false) {
const cacheOptions = options.cache || {
ttl: 300000, // 5 minutes default
maxSize: 500,
};
this.cache = new cache_1.RequestCache(cacheOptions);
}
}
async sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async enforceRateLimit() {
const now = Date.now();
const timeSinceLastRequest = now - this.lastRequestTime;
if (timeSinceLastRequest < this.rateLimitDelay) {
const sleepTime = this.rateLimitDelay - timeSinceLastRequest;
await this.sleep(sleepTime);
}
this.lastRequestTime = Date.now();
}
handleHttpError(response, url) {
const { status, statusText } = response;
switch (status) {
case 401:
throw new exceptions_1.AuthenticationError(`Authentication failed: ${statusText}`);
case 404:
throw new exceptions_1.NotFoundError(`Resource not found: ${statusText}`);
case 429:
throw new exceptions_1.RateLimitError(`Rate limit exceeded: ${statusText}`);
default:
throw new errors_1.RequestError(`HTTP ${status}: ${statusText}`, url, status);
}
}
async get(url, options) {
// Check cache first unless explicitly skipped
if (this.cache && !options?.skipCache) {
const cached = await this.cache.get(url);
if (cached !== null) {
return cached;
}
}
let lastError = null;
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
try {
await this.enforceRateLimit();
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
const response = await (0, node_fetch_1.default)(url, {
method: 'GET',
headers: {
'User-Agent': this.userAgent,
'Accept': 'application/json',
'Accept-Encoding': 'gzip, deflate',
},
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok) {
this.handleHttpError(response, url);
}
const data = await response.json();
// Cache successful responses
if (this.cache) {
await this.cache.set(url, data);
}
return data;
}
catch (error) {
// Handle timeout errors
if (error instanceof Error && error.name === 'AbortError') {
lastError = new errors_1.TimeoutError(url, this.timeout);
}
// Handle network errors
else if (error instanceof Error && (error.message.includes('ENOTFOUND') ||
error.message.includes('ECONNREFUSED') ||
error.message.includes('ETIMEDOUT') ||
error.message.includes('NetworkError'))) {
lastError = new errors_1.NetworkError(error.message, url, error);
}
// Keep existing SecEdgarApiError
else if (error instanceof exceptions_1.SecEdgarApiError) {
lastError = error;
// Don't retry on client errors (4xx) except rate limits
if (error.statusCode &&
error.statusCode >= 400 &&
error.statusCode < 500 &&
error.statusCode !== 429) {
throw error;
}
}
else {
lastError = error;
}
// If this is the last attempt, throw the error
if (attempt === this.maxRetries) {
break;
}
// Exponential backoff for retries
const backoffTime = Math.pow(2, attempt) * 1000;
await this.sleep(backoffTime);
}
}
throw lastError || new exceptions_1.SecEdgarApiError('Max retries exceeded');
}
/**
* Clear the cache
*/
async clearCache() {
if (this.cache) {
await this.cache.invalidateUrl('');
}
}
/**
* Invalidate cache entries for a specific URL pattern
*/
async invalidateCache(urlPattern) {
if (this.cache) {
await this.cache.invalidateUrl(urlPattern);
}
}
}
exports.HttpClient = HttpClient;
//# sourceMappingURL=http-client.js.map