website-scrap-engine
Version:
Configurable website scraper in typescript
43 lines • 1.69 kB
JavaScript
import { adjustConcurrency as logger } from '../logger/logger.js';
export function adjust(downloader) {
var _a;
const { meta } = downloader;
if (!meta.firstPeriodCount) {
meta.firstPeriodCount = downloader.downloadedCount;
meta.lastPeriodTotalCount =
meta.currentPeriodCount =
meta.lastPeriodCount =
meta.firstPeriodCount;
return;
}
const total = downloader.downloadedCount;
meta.lastPeriodCount = meta.currentPeriodCount;
meta.currentPeriodCount = total - meta.lastPeriodTotalCount;
meta.lastPeriodTotalCount = total;
if (downloader.queueSize === 0) {
return logger.info('Queue is empty, keep concurrency as ', downloader.concurrency, 'pending items: ', downloader.queuePending);
}
let concurrency = downloader.concurrency;
if (meta.currentPeriodCount < 2) {
concurrency += 8;
}
else if (meta.currentPeriodCount < meta.lastPeriodCount >> 1) {
concurrency += 4;
}
if (meta.currentPeriodCount < meta.firstPeriodCount >> 2) {
concurrency += 2;
}
if (meta.currentPeriodCount > meta.lastPeriodCount << 2) {
concurrency -= 4;
}
else if (meta.currentPeriodCount > meta.lastPeriodCount << 1) {
concurrency -= 2;
}
else if (meta.currentPeriodCount > meta.firstPeriodCount) {
concurrency -= 2;
}
downloader.concurrency =
Math.max((_a = downloader.options.minConcurrency) !== null && _a !== void 0 ? _a : 4, concurrency);
logger.info('concurrency', downloader.concurrency, 'queue size:', downloader.queueSize);
}
//# sourceMappingURL=adjust-concurrency.js.map