appwrite-utils-cli
Version:
Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.
363 lines (312 loc) • 11.7 kB
text/typescript
import pLimit from "p-limit";
import { logger } from "../../shared/logging.js";
export interface RateLimitConfig {
// Data operations
dataInsertion?: number;
dataUpdate?: number;
dataQuery?: number;
// File operations
fileUpload?: number;
fileDownload?: number;
// Validation operations
validation?: number;
// Relationship operations
relationshipResolution?: number;
// User operations
userCreation?: number;
userUpdate?: number;
// API operations
apiCalls?: number;
}
/**
* Manages rate limiting across the import system.
* Provides configurable p-limit instances for different operation types.
* Builds on existing p-limit usage in attributeManager and other components.
*/
export class RateLimitManager {
private limits: Map<string, any> = new Map();
private config: Required<RateLimitConfig>;
// Default rate limits based on existing usage and best practices
private static readonly DEFAULT_LIMITS: Required<RateLimitConfig> = {
dataInsertion: 5, // Conservative for database writes
dataUpdate: 8, // Slightly higher for updates
dataQuery: 25, // Higher for read operations (from existing queryLimit)
fileUpload: 2, // Very conservative for file uploads
fileDownload: 3, // Slightly higher for downloads
validation: 10, // Higher for validation operations
relationshipResolution: 8, // Moderate for relationship operations
userCreation: 3, // Conservative for user creation
userUpdate: 5, // Moderate for user updates
apiCalls: 15, // General API call limit
};
constructor(config?: Partial<RateLimitConfig>) {
this.config = { ...RateLimitManager.DEFAULT_LIMITS, ...config };
this.initializeLimits();
}
/**
* Initializes p-limit instances for all operation types.
*/
private initializeLimits(): void {
for (const [operation, limit] of Object.entries(this.config)) {
this.limits.set(operation, pLimit(limit));
logger.debug(`Rate limit for ${operation}: ${limit} concurrent operations`);
}
}
/**
* Gets the rate limiter for data insertion operations.
*/
get dataInsertion() {
return this.limits.get('dataInsertion');
}
/**
* Gets the rate limiter for data update operations.
*/
get dataUpdate() {
return this.limits.get('dataUpdate');
}
/**
* Gets the rate limiter for data query operations.
*/
get dataQuery() {
return this.limits.get('dataQuery');
}
/**
* Gets the rate limiter for file upload operations.
*/
get fileUpload() {
return this.limits.get('fileUpload');
}
/**
* Gets the rate limiter for file download operations.
*/
get fileDownload() {
return this.limits.get('fileDownload');
}
/**
* Gets the rate limiter for validation operations.
*/
get validation() {
return this.limits.get('validation');
}
/**
* Gets the rate limiter for relationship resolution operations.
*/
get relationshipResolution() {
return this.limits.get('relationshipResolution');
}
/**
* Gets the rate limiter for user creation operations.
*/
get userCreation() {
return this.limits.get('userCreation');
}
/**
* Gets the rate limiter for user update operations.
*/
get userUpdate() {
return this.limits.get('userUpdate');
}
/**
* Gets the rate limiter for general API calls.
*/
get apiCalls() {
return this.limits.get('apiCalls');
}
/**
* Gets a rate limiter by operation type name.
*
* @param operationType - The type of operation
* @returns The p-limit instance for the operation type
*/
getLimiter(operationType: keyof RateLimitConfig): any {
const limiter = this.limits.get(operationType);
if (!limiter) {
logger.warn(`No rate limiter found for operation type: ${operationType}. Using default API calls limiter.`);
return this.limits.get('apiCalls');
}
return limiter;
}
/**
* Updates the rate limit for a specific operation type.
*
* @param operationType - The operation type to update
* @param newLimit - The new concurrent operation limit
*/
updateLimit(operationType: keyof RateLimitConfig, newLimit: number): void {
this.config[operationType] = newLimit;
this.limits.set(operationType, pLimit(newLimit));
logger.info(`Updated rate limit for ${operationType}: ${newLimit} concurrent operations`);
}
/**
* Updates multiple rate limits at once.
*
* @param newConfig - Partial configuration with new limits
*/
updateLimits(newConfig: Partial<RateLimitConfig>): void {
for (const [operation, limit] of Object.entries(newConfig)) {
if (limit !== undefined) {
this.updateLimit(operation as keyof RateLimitConfig, limit);
}
}
}
/**
* Gets the current configuration.
*/
getConfig(): Required<RateLimitConfig> {
return { ...this.config };
}
/**
* Gets statistics about pending and active operations.
* Useful for monitoring and debugging rate limiting.
*/
getStatistics(): { [operationType: string]: { pending: number; active: number } } {
const stats: { [operationType: string]: { pending: number; active: number } } = {};
for (const [operationType, limiter] of this.limits.entries()) {
stats[operationType] = {
pending: limiter.pendingCount,
active: limiter.activeCount,
};
}
return stats;
}
/**
* Waits for all pending operations to complete.
* Useful for graceful shutdown or testing.
*
* @param timeout - Maximum time to wait in milliseconds (default: 30 seconds)
*/
async waitForCompletion(timeout: number = 30000): Promise<void> {
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
const stats = this.getStatistics();
const hasPendingOperations = Object.values(stats).some(
stat => stat.pending > 0 || stat.active > 0
);
if (!hasPendingOperations) {
logger.info("All rate-limited operations completed");
return;
}
// Log current status
const pendingOperations = Object.entries(stats)
.filter(([_, stat]) => stat.pending > 0 || stat.active > 0)
.map(([type, stat]) => `${type}: ${stat.pending} pending, ${stat.active} active`)
.join(", ");
logger.debug(`Waiting for operations to complete: ${pendingOperations}`);
// Wait a bit before checking again
await new Promise(resolve => setTimeout(resolve, 1000));
}
logger.warn(`Timeout waiting for rate-limited operations to complete after ${timeout}ms`);
}
/**
* Creates a rate limiter with automatic retry logic.
* Combines p-limit with retry functionality for robust operation handling.
*
* @param operationType - The operation type to get the limiter for
* @param maxRetries - Maximum number of retries (default: 3)
* @param retryDelay - Delay between retries in milliseconds (default: 1000)
*/
createRetryLimiter(
operationType: keyof RateLimitConfig,
maxRetries: number = 3,
retryDelay: number = 1000
) {
const limiter = this.getLimiter(operationType);
return async <T>(operation: () => Promise<T>): Promise<T> => {
return limiter(async () => {
let lastError: Error | undefined;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
return await operation();
} catch (error) {
lastError = error as Error;
if (attempt < maxRetries) {
logger.warn(
`Operation failed (attempt ${attempt + 1}/${maxRetries + 1}): ${lastError.message}. Retrying in ${retryDelay}ms...`
);
await new Promise(resolve => setTimeout(resolve, retryDelay));
}
}
}
// If we get here, all retries failed
logger.error(`Operation failed after ${maxRetries + 1} attempts: ${lastError?.message}`);
throw lastError;
});
};
}
/**
* Adjusts rate limits based on API response times and error rates.
* Implements adaptive rate limiting for optimal performance.
*
* @param operationType - The operation type to adjust
* @param responseTime - Average response time in milliseconds
* @param errorRate - Error rate as a percentage (0-100)
*/
adaptiveAdjust(operationType: keyof RateLimitConfig, responseTime: number, errorRate: number): void {
const currentLimit = this.config[operationType];
let newLimit = currentLimit;
// If error rate is high, reduce concurrency
if (errorRate > 10) {
newLimit = Math.max(1, Math.floor(currentLimit * 0.7));
logger.info(`Reducing ${operationType} limit due to high error rate (${errorRate}%): ${currentLimit} -> ${newLimit}`);
}
// If response time is high, reduce concurrency
else if (responseTime > 5000) {
newLimit = Math.max(1, Math.floor(currentLimit * 0.8));
logger.info(`Reducing ${operationType} limit due to high response time (${responseTime}ms): ${currentLimit} -> ${newLimit}`);
}
// If performance is good, gradually increase concurrency
else if (errorRate < 2 && responseTime < 1000 && currentLimit < RateLimitManager.DEFAULT_LIMITS[operationType] * 2) {
newLimit = Math.min(RateLimitManager.DEFAULT_LIMITS[operationType] * 2, currentLimit + 1);
logger.info(`Increasing ${operationType} limit due to good performance: ${currentLimit} -> ${newLimit}`);
}
if (newLimit !== currentLimit) {
this.updateLimit(operationType, newLimit);
}
}
/**
* Resets all rate limits to default values.
*/
resetToDefaults(): void {
logger.info("Resetting all rate limits to default values");
this.config = { ...RateLimitManager.DEFAULT_LIMITS };
this.initializeLimits();
}
/**
* Creates a specialized batch processor with rate limiting.
* Useful for processing large datasets with controlled concurrency.
*
* @param operationType - The operation type for rate limiting
* @param batchSize - Number of items to process in each batch
*/
createBatchProcessor<T, R>(
operationType: keyof RateLimitConfig,
batchSize: number = 50
) {
const limiter = this.getLimiter(operationType);
return async (
items: T[],
processor: (item: T) => Promise<R>,
onProgress?: (processed: number, total: number) => void
): Promise<R[]> => {
const results: R[] = [];
for (let i = 0; i < items.length; i += batchSize) {
const batch = items.slice(i, i + batchSize);
const batchPromises = batch.map(item =>
limiter(() => processor(item))
);
const batchResults = await Promise.allSettled(batchPromises);
for (const result of batchResults) {
if (result.status === 'fulfilled') {
results.push(result.value);
} else {
logger.error(`Batch item failed: ${result.reason}`);
}
}
if (onProgress) {
onProgress(Math.min(i + batchSize, items.length), items.length);
}
}
return results;
};
}
}