notion-page-tree
Version:
Recursively fetch nested Notion pages from the root page/database/block node.
185 lines (184 loc) • 10.5 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createFetchQueueWatcher = exports.createFetchQueue = void 0;
var utils_1 = require("./utils");
var utils_2 = require("../utils");
var flattenEntity_1 = require("./utils/flattenEntity");
var requestChildren_1 = __importDefault(require("./utils/requestChildren"));
var log_1 = require("../utils/log");
/*============================================================================*/
// Fetcher
/*============================================================================*/
/**
* @param maxBlockDepth
* Try to keep this variable to 1.
*
* This is a relative depth between parent page and a block inside it.
* Increasing this value will result in better search indexing and nested child page discovery,
* but it will also increase request count (more than) EXPONENTIALLY.
* Notion API may throw error as it has broader request rate limits.
* So if you want to discover nested pages, don't put them under plain blocks, rather put them on the plain level of the parent page.
* And if you want better search indexing, put search keywords on the root level of the page or put them on the page property.
*/
var createFetchQueue = function (_a) {
var requestParameters = _a.requestParameters, _b = _a.maxConcurrency, maxConcurrency = _b === void 0 ? 3 : _b, _c = _a.maxRequestDepth, maxRequestDepth = _c === void 0 ? 3 : _c, _d = _a.maxBlockDepth, maxBlockDepth = _d === void 0 ? 2 : _d, _e = _a.maxRetry, maxRetry = _e === void 0 ? 3 : _e, _f = _a.databaseQueryFilter, databaseQueryFilter = _f === void 0 ? undefined : _f;
var currentMaxDepth = 0;
var currentRequestCount = 0;
var errorTimeout = undefined;
// initiate collections
var request_ready_queue = [];
var request_promise_queue = [];
var page_collection = {};
var rootEntity = {
id: requestParameters.entry_id,
type: requestParameters.entry_type,
depth: 0,
blockContentPlainText: '',
children: []
};
// add root to ready queue
request_ready_queue.push({
parentToAssign: rootEntity,
parentToRequest: rootEntity,
retry: 0
});
// add root to page collection
page_collection[rootEntity.id] = (0, flattenEntity_1.flattenEntity)(rootEntity);
/*============================================================================*/
// Job 1 : ready queue => promise queue
/*============================================================================*/
var requestReadyRoutine = setInterval(function () {
if (request_ready_queue.length > 0 &&
request_promise_queue.length < maxConcurrency) {
var slots = maxConcurrency - request_promise_queue.length;
var new_requests = request_ready_queue.splice(0, slots);
var new_promises = new_requests.map(function (request) { return ({
parentToAssign: request.parentToAssign,
parentToRequest: request.parentToRequest,
children: (0, utils_2.makeQueryablePromise)((0, requestChildren_1.default)(request.parentToRequest, requestParameters, databaseQueryFilter)),
retry: request.retry
}); });
currentRequestCount += new_promises.length;
request_promise_queue.push.apply(request_promise_queue, new_promises);
}
}, 350); // Rate limit : 3 requests per second
/*============================================================================*/
// Job 2 : promise queue => handle setteled
/*============================================================================*/
var requestPromiseRoutine = setInterval(function () {
if (request_promise_queue.length > 0) {
// for each responded child
request_promise_queue.forEach(function (promise, idx) {
// handle [ rejected ]
if (promise.children.isRejected()) {
var rate_limited = promise.children.getRejectedValue().code === 'rate_limited';
if (!rate_limited) {
(0, log_1.stdout)("Retrying rejected request.");
promise.retry < maxRetry
? request_ready_queue.unshift({
parentToAssign: promise.parentToAssign,
parentToRequest: promise.parentToRequest,
retry: promise.retry + 1
})
: (0, log_1.stderr)("Max retry count reached requesting children of ".concat(promise.parentToRequest.type, " ").concat(promise.parentToRequest.id, "."));
// remove from promise queue
request_promise_queue.splice(idx, 1);
}
if (rate_limited) {
(0, log_1.stdout)("Flushing promise queue and waiting 3 minutes from now...");
var previousConcurrency_1 = maxConcurrency;
maxConcurrency = 0;
request_ready_queue.push.apply(request_ready_queue, request_promise_queue.splice(idx, 1).map(function (promise) { return ({
parentToAssign: promise.parentToAssign,
parentToRequest: promise.parentToRequest,
retry: promise.retry
}); }));
// reset error delay timer
errorTimeout && clearTimeout(errorTimeout);
errorTimeout = setTimeout(function () {
(0, log_1.stdout)("Restarting request...");
maxConcurrency = previousConcurrency_1;
}, 1000 * 60 * 3);
}
}
// handle [ fulfilled ]
if (promise.children.isFulfilled()) {
promise.children.getResolvedValue().forEach(function (child) {
// update max depth, report
if (currentMaxDepth < child.depth) {
currentMaxDepth = child.depth;
(0, log_1.stdout)("Request depth reached ".concat(currentMaxDepth, ", with ").concat(Object.keys(page_collection).length, " pages traversed, ").concat(currentRequestCount, " requests made, and ").concat(request_ready_queue.length, " left to be requested."));
}
// when child is page or database
if (child.type === 'database' || child.type === 'page') {
// push child to parent
promise.parentToAssign.children.push(child);
page_collection[promise.parentToAssign.id].children.push(child.id);
// add to page collection
page_collection[child.id] = (0, flattenEntity_1.flattenEntity)(child);
// push new requests
child.depth < maxRequestDepth && // (if not reached page depth limit)
request_ready_queue.push({
parentToAssign: child,
parentToRequest: child,
retry: 0
});
}
// when child is block (not page or database)
if (child.type === 'block') {
var plainText = (0, utils_1.extractPlainTextFromBlock)(child.metadata);
page_collection[promise.parentToAssign.id].blockContentPlainText += "".concat(plainText, " ");
// push new requests
child.depth - promise.parentToAssign.depth < maxBlockDepth && // (if not reached relative block depth limit)
child.depth < maxRequestDepth && // (if not reached page depth limit)
utils_1.blocksWithoutChildren.some(function (type) { return type !== child.metadata.type; }) && // (if the block can have children)
request_ready_queue.push({
// block content should be assigned to the nearest parent
parentToAssign: promise.parentToAssign,
parentToRequest: child,
retry: 0
});
}
});
// remove from promise queue
request_promise_queue.splice(idx, 1);
}
});
}
});
return {
requestReadyRoutine: requestReadyRoutine,
requestPromiseRoutine: requestPromiseRoutine,
request_ready_queue: request_ready_queue,
request_promise_queue: request_promise_queue,
page_collection: page_collection,
rootEntity: rootEntity
};
};
exports.createFetchQueue = createFetchQueue;
/*============================================================================*/
// Fetcher Watcher
/*============================================================================*/
function createFetchQueueWatcher(_a) {
var requestReadyRoutine = _a.requestReadyRoutine, requestPromiseRoutine = _a.requestPromiseRoutine, request_ready_queue = _a.request_ready_queue, request_promise_queue = _a.request_promise_queue, page_collection = _a.page_collection, rootEntity = _a.rootEntity;
(0, log_1.stdout)("Waiting for fetch queue to resolve...");
// check for routines
return new Promise(function (resolve) {
var fetchQueueWatcher = setInterval(function () {
if (request_promise_queue.length === 0 &&
request_ready_queue.length === 0) {
clearInterval(requestPromiseRoutine);
clearInterval(requestReadyRoutine);
clearInterval(fetchQueueWatcher);
resolve({
page_collection: page_collection,
root: rootEntity
});
}
}, 1000);
});
}
exports.createFetchQueueWatcher = createFetchQueueWatcher;