notion-page-tree
Version:
Recursively fetch nested Notion pages from the root page/database/block node.
329 lines (328 loc) • 18.7 kB
JavaScript
"use strict";
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
var path_1 = __importDefault(require("path"));
var fetcher_1 = require("./fetcher");
var createPageSearchIndex_1 = require("./server/utils/createPageSearchIndex");
var promises_1 = require("fs/promises");
var setupServer_1 = require("./server/setupServer");
var fsExists_1 = __importDefault(require("./utils/fsExists"));
var client_1 = require("@notionhq/client");
var utils_1 = require("./utils");
var appendToDotEnv_1 = require("./utils/appendToDotEnv");
var dotenv_1 = __importDefault(require("dotenv"));
var os_1 = require("os");
var log_1 = require("./utils/log");
var convertNotionId_1 = require("./server/utils/convertNotionId");
var NotionPageTree = /** @class */ (function () {
function NotionPageTree(options) {
this.requestParameters = {
client: new client_1.Client(),
entry_id: undefined,
entry_key: undefined,
entry_type: undefined
};
this.private_file_path = (options === null || options === void 0 ? void 0 : options.private_file_path) || './';
this.createFetchQueueOptions = (options === null || options === void 0 ? void 0 : options.createFetchQueueOptions) || {};
this.searchIndexing = (options === null || options === void 0 ? void 0 : options.searchIndexing) || true;
this.data_set = {
page_collection: undefined,
root: undefined,
search_index: undefined,
search_suggestion: undefined
};
}
/**
* Find cached documents to serve immediately.
*/
NotionPageTree.prototype.parseCachedDocument = function () {
return __awaiter(this, void 0, void 0, function () {
var _a, readFilePath, treeFileExists, collectionFileExists, _b, _c, _d, _e, _f, _g, _h, idx, tkn;
var _this = this;
return __generator(this, function (_j) {
switch (_j.label) {
case 0: return [4 /*yield*/, (0, fsExists_1.default)(this.private_file_path)];
case 1:
if (!
// If directory doesn't exist, create
(_j.sent()))
// If directory doesn't exist, create
return [3 /*break*/, 3];
return [4 /*yield*/, (0, promises_1.lstat)(this.private_file_path)];
case 2:
_a = (_j.sent()).isDirectory()
? true
: function () {
throw new Error("Path ".concat(_this.private_file_path, " is not a directory."));
};
return [3 /*break*/, 5];
case 3: return [4 /*yield*/, (0, promises_1.mkdir)(this.private_file_path)];
case 4:
_a = _j.sent();
_j.label = 5;
case 5:
// If directory doesn't exist, create
_a;
return [4 /*yield*/, (0, promises_1.readdir)(this.private_file_path)];
case 6:
readFilePath = _j.sent();
treeFileExists = readFilePath.some(function (fileName) { return fileName === 'tree.json'; });
collectionFileExists = readFilePath.some(function (fileName) { return fileName === 'collection.json'; });
if (!(treeFileExists && collectionFileExists)) return [3 /*break*/, 11];
// Read and parse files
_b = this.data_set;
_d = (_c = JSON).parse;
return [4 /*yield*/, (0, promises_1.readFile)(path_1.default.resolve(this.private_file_path, 'collection.json'))];
case 7:
// Read and parse files
_b.page_collection = _d.apply(_c, [(_j.sent()).toString()]);
_e = this.data_set;
_g = (_f = JSON).parse;
return [4 /*yield*/, (0, promises_1.readFile)(path_1.default.resolve(this.private_file_path, 'tree.json'))];
case 8:
_e.root = _g.apply(_f, [(_j.sent()).toString()]);
if (!this.searchIndexing) return [3 /*break*/, 10];
_h = (0, createPageSearchIndex_1.createPageSearchIndex)(this.data_set.page_collection, ['description', 'curation']), idx = _h.idx, tkn = _h.tkn;
this.data_set.search_index = idx;
this.data_set.search_suggestion = tkn;
return [4 /*yield*/, (0, promises_1.writeFile)(path_1.default.resolve(this.private_file_path, 'search-suggestion.json'), JSON.stringify(JSON.parse(JSON.stringify(idx.toJSON())).invertedIndex.map(function (tokenSet) { return tokenSet[0]; })))];
case 9:
_j.sent();
_j.label = 10;
case 10: return [3 /*break*/, 12];
case 11:
(0, log_1.stdout)("No previously fetched file. Page data variables will be undefined until the first fetch is completed.");
_j.label = 12;
case 12: return [2 /*return*/];
}
});
});
};
/**
* Look for environment variables that are needed to fetch page tree.
* If they don't exist, ask for it.
*/
/**
* Look for environment variables that are needed to fetch page tree.
* @param prompt If variables are missing, prompt user with interactive CLI.
* @param forceRewrite Prompt user with interactive CLI even if variables exist.
* @returns
* client: Notion API client instance.
*
* entry_id: Root entry's id (any block is allowed).
*
* entry_key: Root entry's integration key. Create one here (https://www.notion.so/my-integrations). ⚠️ Must be added to the entry block's "share" settings.
*
* entry_type: 'page' | 'database' | 'block'(all the others)
*/
NotionPageTree.prototype.setRequestParameters = function (_a) {
var _b = _a.prompt, prompt = _b === void 0 ? false : _b, _c = _a.forceRewrite, forceRewrite = _c === void 0 ? false : _c;
return __awaiter(this, void 0, void 0, function () {
var envFilePath, envFile, _d, _e, _f, _g, _h, _j, _k;
return __generator(this, function (_l) {
switch (_l.label) {
case 0:
envFilePath = path_1.default.resolve('.env');
envFile = dotenv_1.default.config({ path: envFilePath, override: true }).parsed;
// Try to read from process.env.
process.env.NOTION_ENTRY_ID &&
(this.requestParameters.entry_id = process.env.NOTION_ENTRY_ID);
process.env.NOTION_ENTRY_KEY &&
(this.requestParameters.entry_key = process.env.NOTION_ENTRY_KEY);
process.env.NOTION_ENTRY_TYPE &&
(this.requestParameters.entry_type = process.env
.NOTION_ENTRY_TYPE);
if (!prompt) return [3 /*break*/, 7];
_d = (this.requestParameters.entry_id === undefined || forceRewrite) &&
prompt;
if (!_d) return [3 /*break*/, 2];
_e = this.requestParameters;
_f = convertNotionId_1.convertNotionId;
return [4 /*yield*/, (0, utils_1.askInput)('entry_id', "Enter root block's id", true)];
case 1:
_d = (_e.entry_id = _f.apply(void 0, [_l.sent(), 'dashed']));
_l.label = 2;
case 2:
_d;
_g = (this.requestParameters.entry_key === undefined || forceRewrite);
if (!_g) return [3 /*break*/, 4];
_h = this.requestParameters;
return [4 /*yield*/, (0, utils_1.askInput)('entry_key', "Enter root block's key", true)];
case 3:
_g = (_h.entry_key = _l.sent());
_l.label = 4;
case 4:
_g;
_j = (this.requestParameters.entry_type === undefined || forceRewrite);
if (!_j) return [3 /*break*/, 6];
_k = this.requestParameters;
return [4 /*yield*/, (0, utils_1.askSelect)(['database', 'page', 'block'], 'select entry block"s type')];
case 5:
_j = (_k.entry_type =
_l.sent());
_l.label = 6;
case 6:
_j;
// write new variables to package's .env file
this.requestParameters.entry_id &&
this.requestParameters.entry_key &&
this.requestParameters.entry_type &&
(0, appendToDotEnv_1.appendToDotEnv)(envFilePath, {
NOTION_ENTRY_ID: this.requestParameters.entry_id,
NOTION_ENTRY_KEY: this.requestParameters.entry_key,
NOTION_ENTRY_TYPE: this.requestParameters.entry_type
}, envFile ? envFile : undefined);
(0, log_1.stdout)("Reqparams written in .env file".concat(os_1.EOL, "id : ").concat(this.requestParameters.entry_id).concat(os_1.EOL, "key : ").concat(this.requestParameters.entry_key).concat(os_1.EOL, "type : ").concat(this.requestParameters.entry_type));
_l.label = 7;
case 7:
// if still doesn't exist, throw
if (this.requestParameters.entry_id === undefined ||
this.requestParameters.entry_key === undefined ||
this.requestParameters.entry_type === undefined) {
throw new Error("Can't find environment variables.".concat(os_1.EOL, "Set ").concat(this.requestParameters.entry_id === '' ? "NOTION_ENTRY_ID".concat(os_1.EOL) : '').concat(this.requestParameters.entry_key === ''
? "NOTION_ENTRY_KEY".concat(os_1.EOL)
: '').concat(this.requestParameters.entry_type === ''
? "NOTION_ENTRY_TYPE".concat(os_1.EOL)
: '', "in .env file.").concat(os_1.EOL).concat(os_1.EOL));
}
else {
// else, return
return [2 /*return*/, this.requestParameters];
}
return [2 /*return*/];
}
});
});
};
/**
* Fetch pages asynchronously, and then assign results to variables.
*/
NotionPageTree.prototype.fetchOnce = function () {
return __awaiter(this, void 0, void 0, function () {
var fetchQueue, fetchResult, _a, idx, tkn;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
if (!(this.requestParameters.entry_id === undefined ||
this.requestParameters.entry_key === undefined ||
this.requestParameters.entry_type === undefined)) return [3 /*break*/, 2];
return [4 /*yield*/, this.setRequestParameters({ prompt: false, forceRewrite: false })];
case 1:
_b.sent();
_b.label = 2;
case 2:
fetchQueue = (0, fetcher_1.createFetchQueue)(__assign(__assign({}, this.createFetchQueueOptions), { requestParameters: this.requestParameters }));
return [4 /*yield*/, (0, fetcher_1.createFetchQueueWatcher)(fetchQueue)];
case 3:
fetchResult = _b.sent();
this.data_set.page_collection = fetchResult.page_collection;
this.data_set.root = fetchResult.root;
// write fetch results to local disk for caching
return [4 /*yield*/, (0, promises_1.writeFile)(path_1.default.resolve(this.private_file_path, 'collection.json'), JSON.stringify(this.data_set.page_collection))];
case 4:
// write fetch results to local disk for caching
_b.sent();
return [4 /*yield*/, (0, promises_1.writeFile)(path_1.default.resolve(this.private_file_path, 'tree.json'), JSON.stringify(this.data_set.root))];
case 5:
_b.sent();
if (!this.searchIndexing) return [3 /*break*/, 7];
_a = (0, createPageSearchIndex_1.createPageSearchIndex)(this.data_set.page_collection, ['description', 'curation']), idx = _a.idx, tkn = _a.tkn;
this.data_set.search_index = idx;
this.data_set.search_suggestion = tkn;
return [4 /*yield*/, (0, promises_1.writeFile)(path_1.default.resolve(this.private_file_path, 'search-suggestion.json'), JSON.stringify(JSON.parse(JSON.stringify(idx.toJSON())).invertedIndex.map(function (tokenSet) { return tokenSet[0]; })))];
case 6:
_b.sent();
_b.label = 7;
case 7: return [2 /*return*/];
}
});
});
};
/**
* Create asynchronouse fetch loop, which waits for `fetchIntervalMinutes` after each fetch is completed.
*/
NotionPageTree.prototype.startFetchLoop = function (fetchInterval) {
return __awaiter(this, void 0, void 0, function () {
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.fetchOnce()];
case 1:
_a.sent();
(0, log_1.stdout)("Waiting ".concat(fetchInterval, " milliseconds for the next fetch."));
this.fetchLoopTimer = setTimeout(function () {
_this.startFetchLoop(fetchInterval);
}, fetchInterval);
return [2 /*return*/];
}
});
});
};
/**
* Stop the fetch loop after current fetch is resolved.
*/
NotionPageTree.prototype.stopFetchLoop = function () {
this.fetchLoopTimer
? clearTimeout(this.fetchLoopTimer)
: (0, log_1.stderr)('fetch loop is not started yet');
};
/**
* Setup servers that serves fetched pages and its search indexes. See details in readme.
*/
NotionPageTree.prototype.setupServer = function (_a) {
var port = _a.port;
// create server
return (0, setupServer_1.setupServer)(port, this.data_set);
};
return NotionPageTree;
}());
exports.default = NotionPageTree;