@flatfile/safe-api
Version:
Flatfile Safe API client with streaming capabilities
81 lines (80 loc) • 2.43 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.PaginatedCollection = void 0;
class PaginatedCollection {
constructor(factory, options) {
var _a;
this.factory = factory;
this.options = options;
/**
* Default page size of 1000
* @private
*/
this.pageSize = 1000;
/**
* Get all pages as fast as the rate limiter allows vs default sequential
*/
this._asap = false;
if ((_a = this.options) === null || _a === void 0 ? void 0 : _a.pageSize) {
this.pageSize = this.options.pageSize;
}
}
/**
* Provide a progress tracking callback
*/
track(tracker) {
this._tracker = tracker;
}
asap() {
this._asap = true;
}
/**
* Do something with each record as they're loaded (can help save memory)
*/
map(cb) {
return this.run((records) => {
return records.map(cb);
});
}
/**
* Wait for all records (this will pool in memory, be careful)
*/
all() {
return this.run();
}
async run(cb) {
var _a;
if ((_a = this.options) === null || _a === void 0 ? void 0 : _a.getCounts) {
const count = await this.options.getCounts();
const pages = Math.ceil(count / this.pageSize);
const promises = Array.from({ length: pages }, (_, i) => this.factory(i + 1, this.pageSize).then((r) => (cb ? cb(r) : r)));
const results = await Promise.all(promises);
return results.flat();
}
else {
const out = [];
let page = 1;
while (true) {
const section = await this.factory(page++, this.pageSize).then((r) => (cb ? cb(r) : r));
if (!Array.isArray(section)) {
break;
}
out.push(section);
if (!(section === null || section === void 0 ? void 0 : section.length) || section.length < this.pageSize) {
break;
}
}
return out.flat();
}
}
/**
* When awaited, actually run the stuff
*
* @param resolve
* @param reject
*/
then(resolve, reject) {
return this.run().then(resolve, reject);
}
}
exports.PaginatedCollection = PaginatedCollection;