instamancer
Version:
Scrape the Instagram API with Puppeteer
79 lines • 5.37 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.GetPool = void 0;
class GetJob {
constructor(url, name, extension, downloadUpload) {
this.finished = false;
this.url = url;
this.name = name;
this.extension = extension;
this.downloadUpload = downloadUpload;
}
async start() {
await this.downloadUpload(this.url, this.name, this.extension);
this.finished = true;
}
}
/**
* A pool of jobs that only executes k jobs 'simultaneously'
*/
class GetPool {
constructor(connections = 1, downloadUpload) {
// Job promises
this.promises = [];
// Jobs that are currently being executed
this.runningJobs = [];
// Jobs that are yet to be executed
this.queuedJobs = [];
// Lock loop function execution
this.lock = false;
// End-of-input signal triggered externally by close()
this.finished = false;
this.maxConnections = connections;
this.loop = setInterval(() => {
this.poolLoop.bind(this)();
}, 100);
this.downloadUpload = downloadUpload;
}
add(url, name, extension) {
this.queuedJobs.push(new GetJob(url, name, extension, this.downloadUpload));
}
close(resolve) {
this.finished = true;
this.resolve = resolve;
}
poolLoop() {
// Obtain lock or cancel
if (this.lock) {
return;
}
else {
this.lock = true;
}
// Remove finished jobs
for (let i = 0; i < this.runningJobs.length; i++) {
if (this.runningJobs[i].finished) {
this.runningJobs.splice(i);
i = 0;
}
}
// Add new jobs to empty running slots
while (this.queuedJobs.length > 0 &&
this.runningJobs.length < this.maxConnections) {
const job = this.queuedJobs.shift();
this.promises.push(job.start());
this.runningJobs.push(job);
}
// End the interval when end-of-input signal given
if (this.finished &&
this.queuedJobs.length === 0 &&
this.runningJobs.length === 0) {
clearInterval(this.loop);
this.resolve();
}
// Release lock
this.lock = false;
}
}
exports.GetPool = GetPool;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2V0UG9vbC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbImdldFBvb2wudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBRUEsTUFBTSxNQUFNO0lBV1IsWUFBWSxHQUFXLEVBQUUsSUFBWSxFQUFFLFNBQWlCLEVBQUUsY0FBYztRQVZqRSxhQUFRLEdBQVksS0FBSyxDQUFDO1FBVzdCLElBQUksQ0FBQyxHQUFHLEdBQUcsR0FBRyxDQUFDO1FBQ2YsSUFBSSxDQUFDLElBQUksR0FBRyxJQUFJLENBQUM7UUFDakIsSUFBSSxDQUFDLFNBQVMsR0FBRyxTQUFTLENBQUM7UUFDM0IsSUFBSSxDQUFDLGNBQWMsR0FBRyxjQUFjLENBQUM7SUFDekMsQ0FBQztJQUVNLEtBQUssQ0FBQyxLQUFLO1FBQ2QsTUFBTSxJQUFJLENBQUMsY0FBYyxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLElBQUksRUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUM7UUFDL0QsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLENBQUM7SUFDekIsQ0FBQztDQUNKO0FBRUQ7O0dBRUc7QUFDSCxNQUFhLE9BQU87SUFrQ2hCLFlBQ0ksY0FBc0IsQ0FBQyxFQUN2QixjQUlrQjtRQXZDdEIsZUFBZTtRQUNSLGFBQVEsR0FBeUIsRUFBRSxDQUFDO1FBRTNDLHlDQUF5QztRQUNqQyxnQkFBVyxHQUFhLEVBQUUsQ0FBQztRQUVuQyxtQ0FBbUM7UUFDM0IsZUFBVSxHQUFhLEVBQUUsQ0FBQztRQVFsQywrQkFBK0I7UUFDdkIsU0FBSSxHQUFZLEtBQUssQ0FBQztRQUU5QixzREFBc0Q7UUFDOUMsYUFBUSxHQUFZLEtBQUssQ0FBQztRQXNCOUIsSUFBSSxDQUFDLGNBQWMsR0FBRyxXQUFXLENBQUM7UUFDbEMsSUFBSSxDQUFDLElBQUksR0FBRyxXQUFXLENBQUMsR0FBRyxFQUFFO1lBQ3pCLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7UUFDL0IsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1FBQ1IsSUFBSSxDQUFDLGNBQWMsR0FBRyxjQUFjLENBQUM7SUFDekMsQ0FBQztJQUVNLEdBQUcsQ0FBQyxHQUFXLEVBQUUsSUFBWSxFQUFFLFNBQWlCO1FBQ25ELElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUNoQixJQUFJLE1BQU0sQ0FBQyxHQUFHLEVBQUUsSUFBSSxFQUFFLFNBQVMsRUFBRSxJQUFJLENBQUMsY0FBYyxDQUFDLENBQ3hELENBQUM7SUFDTixDQUFDO0lBRU0sS0FBSyxDQUFDLE9BQU87UUFDaEIsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLENBQUM7UUFDckIsSUFBSSxDQUFDLE9BQU8sR0FBRyxPQUFPLENBQUM7SUFDM0IsQ0FBQztJQUVPLFFBQVE7UUFDWix3QkFBd0I7UUFDeEIsSUFBSSxJQUFJLENBQUMsSUFBSSxFQUFFO1lBQ1gsT0FBTztTQUNWO2FBQU07WUFDSCxJQUFJLENBQUMsSUFBSSxHQUFHLElBQUksQ0FBQztTQUNwQjtRQUVELHVCQUF1QjtRQUN2QixLQUFLLElBQUksQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEdBQUcsSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxFQUFFLEVBQUU7WUFDOUMsSUFBSSxJQUFJLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDLFFBQVEsRUFBRTtnQkFDOUIsSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLENBQUM7Z0JBQzNCLENBQUMsR0FBRyxDQUFDLENBQUM7YUFDVDtTQUNKO1FBRUQsc0NBQXNDO1FBQ3RDLE9BQ0ksSUFBSSxDQUFDLFVBQVUsQ0FBQyxNQUFNLEdBQUcsQ0FBQztZQUMxQixJQUFJLENBQUMsV0FBVyxDQUFDLE1BQU0sR0FBRyxJQUFJLENBQUMsY0FBYyxFQUMvQztZQUNFLE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxVQUFVLENBQUMsS0FBSyxFQUFFLENBQUM7WUFDcEMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUM7WUFDaEMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7U0FDOUI7UUFFRCxrREFBa0Q7UUFDbEQsSUFDSSxJQUFJLENBQUMsUUFBUTtZQUNiLElBQUksQ0FBQyxVQUFVLENBQUMsTUFBTSxLQUFLLENBQUM7WUFDNUIsSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUMvQjtZQUNFLGFBQWEsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7WUFDekIsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1NBQ2xCO1FBRUQsZUFBZTtRQUNmLElBQUksQ0FBQyxJQUFJLEdBQUcsS0FBSyxDQUFDO0lBQ3RCLENBQUM7Q0FDSjtBQW5HRCwwQkFtR0MifQ==