@jitl/notion-api
Version:
The missing companion library for the official Notion public API.
808 lines • 29.9 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getCustomPropertyValue = exports.getPageTitle = exports.defaultSlug = exports.CMSPropertyResolver = exports.CMS = void 0;
const tslib_1 = require("tslib");
/**
* This file introduces a [[CMS]] - a bare-bones, read-only headless content
* management system - on top of the pages in a Notion database.
* @category CMS
* @module
*/
/* eslint-disable @typescript-eslint/ban-types */
const path = tslib_1.__importStar(require("path"));
const fsOld = tslib_1.__importStar(require("fs"));
const util_1 = require("@jitl/util");
const notion_api_1 = require("./notion-api");
const assets_1 = require("./assets");
const cache_1 = require("./cache");
const backlinks_1 = require("./backlinks");
const src_1 = require("@notionhq/client/build/src");
const query_1 = require("./query");
const DEBUG_CMS = notion_api_1.DEBUG.extend('cms');
const fs = fsOld.promises;
const DEBUG_SLUG = DEBUG_CMS.extend('slug');
/**
* A Content Management System (CMS) based on the Notion API.
* Each CMS instance wraps a single Notion database that contains [[CMSPage]]s.
* Pages and their contents loaded from the CMS are cached in-memory, and
* optionally on disk.
*
* See [[CMSConfig]] for configuration options.
*
* ```typescript
* const cms = new CMS({
* notion,
* database_id: 'example',
* slug: 'productCode',
* visible: 'publicAccess',
* schema: inferDatabaseSchema({
* productCode: {
* name: 'Product Code',
* type: 'rich_text',
* },
* Subtitle: {
* type: 'rich_text',
* },
* publicAccess: {
* name: 'Public Access',
* type: 'checkbox',
* },
* Date: {
* type: 'date',
* },
* }),
* getFrontmatter: ({ properties }) => ({
* ...properties,
* productCode: richTextAsPlainText(properties.productCode),
* }),
* });
*
* const oldPagesIterator = cms.query({
* filter: cms.filter.and(
* cms.filter.Date.before('2020-01-01'),
* cms.filter.publicAccess.equals(true)
* ),
* sorts: [cms.sort.last_edited_time.descending],
* });
*
* const drafts = cms.scope({
* filter: cms.getVisibleEqualsFilter(false),
* showInvisible: true,
* });
*
* const draftsIterator = drafts.query({})
* ```
*
* @category CMS
*/
class CMS {
constructor(config) {
this.config = config;
/** Indexes links between the pages that have been loaded into memory. */
this.backlinks = new backlinks_1.Backlinks();
/**
* Indexes Notion API objects in pages that have been loaded into memory.
*/
this.notionObjects = new cache_1.NotionObjectIndex();
/** Maps from Page ID to CMSPage */
this.pages = new Map();
/** Asset downloader, requires `assets` configuration */
this.assets = this.config.assets && new AssetCache(this.config.assets);
/**
* Filter helpers for this CMS's database schema.
*
* ```typescript
* const cms = new CMS({ ... })
* cms.query({
* filter: cms.filter.and(
* cms.filter.createdAt.last_week({}),
* cms.filter.featured.equals(true)
* ),
* })
* ```
*/
this.filter = (0, query_1.databaseFilterBuilder)(this.schema);
/**
* Sort helpers for this CMS's database schema.
*
* ```typescript
* const cms = new CMS({ ... })
* cms.query({
* sorts: [
* cms.sort.createdAt.descending,
* cms.sort.title.ascending,
* ],
* })
* ```
*/
this.sort = (0, query_1.databaseSortBuilder)(this.schema);
/** Private for now, because the semantics may change. */
this.pageContentCache = new PageContentCache(this.config.cache);
this.customPropertyFilters = {};
const defaultScope = this.scope({});
this.query = defaultScope.query;
this.propertyResolver = new CMSPropertyResolver(this);
}
/**
* See also [[CMSConfig.schema]].
*/
get schema() {
return this.config.schema;
}
/** Retrieve a CMS page by ID. */
async loadPageById(pageId, options = {}) {
let page;
let cached;
try {
cached = await this.pageContentCache.getPageContent(this.config.notion, pageId);
page = cached.page || (await this.config.notion.pages.retrieve({ page_id: pageId }));
}
catch (error) {
if ((0, src_1.isNotionClientError)(error) && error.code === src_1.APIErrorCode.ObjectNotFound) {
return undefined;
}
throw error;
}
if (!(0, notion_api_1.isFullPage)(page)) {
return undefined;
}
const visible = options.showInvisible || (await this.getVisible(page));
if (!visible) {
return undefined;
}
const cmsPage = await this.buildCMSPage({
children: cached.children,
page,
reIndexChildren: !cached.hit,
});
return cmsPage;
}
/**
* Retrieve a CMS page by its slug.
*
* Note that configuring the CMS to use a property for the slug is more
* efficient than using a derived function, which requires a O(n) scan of the
* database.
*/
async loadPageBySlug(slug, options = {}) {
// Optimization - the default slug is just the page ID (without dashes),
// so we can just load by ID.
if (this.config.slug === undefined) {
DEBUG_SLUG('not configured, loading by ID: %s', slug);
return this.loadPageById(slug, options);
}
// Optimization - empty slugs fall back to page ID, so maybe it's easier to load by ID.
if (slug.length === 32) {
try {
DEBUG_SLUG('length = 32, try loading by ID: %s', slug);
const byId = await this.loadPageById(slug, options);
if (byId) {
return byId;
}
}
catch (error) {
// Ignore
}
}
return this.findPageWithSlugRaw({
slug,
options,
queryParameters: this.getQueryParameters({
...options,
slug,
}),
});
}
/**
* *Raw* - prefer to use [[loadPageBySlug]] instead.
* Scan the requests of `queryParameters` for the first page with the given slug.
*/
async findPageWithSlugRaw(args) {
const { slug, options, queryParameters } = args;
DEBUG_SLUG('query for slug %s: %o', slug, queryParameters.filter);
for await (const page of (0, notion_api_1.iteratePaginatedAPI)(this.config.notion.databases.query, queryParameters)) {
if ((0, notion_api_1.isFullPage)(page)) {
const pageSlug = await this.getSlug(page);
DEBUG_SLUG('scan page %s: has slug %s', page.id, pageSlug);
if (pageSlug === slug) {
const visible = options.showInvisible || (await this.getVisible(page));
if (!visible) {
DEBUG_SLUG('scan page %s: not visible');
return undefined;
}
const cached = await this.pageContentCache.getPageContent(this.config.notion, page);
const cmsPage = await this.buildCMSPage({
children: cached.children,
slug,
page,
reIndexChildren: !cached.hit,
});
return cmsPage;
}
}
}
}
/**
* *Raw* - prefer to use [[query]] instead.
* Scan the results of `queryParameters` and return each page as a [[CMSPage]].
*/
async *queryRaw(args) {
const { queryParameters, options } = args;
for await (const page of (0, notion_api_1.iteratePaginatedAPI)(this.config.notion.databases.query, queryParameters)) {
if ((0, notion_api_1.isFullPage)(page)) {
const visible = options.showInvisible || (await this.getVisible(page));
if (!visible) {
continue;
}
const cached = await this.pageContentCache.getPageContent(this.config.notion, page);
const cmsPage = await this.buildCMSPage({
children: cached.children,
page,
reIndexChildren: !cached.hit,
});
yield cmsPage;
}
}
}
getQueryParameters(args = {}) {
const { slug, showInvisible } = args;
const visibleFilter = showInvisible ? undefined : this.getVisibleEqualsFilter(true);
const slugFilter = slug === undefined ? undefined : this.getSlugEqualsFilter(slug);
return {
database_id: this.config.database_id,
filter: this.filter.and(visibleFilter, slugFilter),
sorts: this.getDefaultSorts(),
};
}
async downloadAssets(cmsPage) {
const assetCache = this.assets;
if (!assetCache) {
return;
}
const assetRequests = [];
const enqueue = (req) => assetRequests.push(req);
enqueue({
object: 'page',
id: cmsPage.content.id,
field: 'icon',
});
enqueue({
object: 'page',
id: cmsPage.content.id,
field: 'cover',
});
(0, notion_api_1.visitChildBlocks)(cmsPage.content.children, (block) => {
if (block.type === 'image') {
enqueue({
object: 'block',
id: block.id,
field: 'image',
});
}
if (block.type === 'callout') {
enqueue({
object: 'block',
id: block.id,
field: 'icon',
});
}
});
// TODO: concurrency limit
await Promise.all(assetRequests.map((request) => assetCache.downloadAssetRequest({
cache: this.notionObjects,
notion: this.config.notion,
request,
})));
}
scope(args) {
return this.createScope({
...args,
parentScope: this,
});
}
createScope(args) {
const { parentScope, filter, sorts, ...retrieveOptions } = args;
const getQueryParameters = (args = {}) => (0, query_1.extendQueryParameters)(parentScope.getQueryParameters({
...retrieveOptions,
...args,
}), {
filter,
sorts,
});
const childScope = {
query: (args, options) => this.queryRaw({
queryParameters: (0, query_1.extendQueryParameters)(getQueryParameters(options), args || {}),
options: options || {},
}),
getQueryParameters,
scope: (args) => this.createScope({
...args,
parentScope: childScope,
}),
};
return childScope;
}
/**
* If `config.visible` is a property pointer, return a filter for `visibleProperty = isVisible`.
* Note that you must also set `showInvisible: true` for query APIs to return invisible pages,
* otherwise they will be filtered out in-memory.
*
* This filter is added automatically to queries in the CMS and
* [[getQueryParameters]] unless their `showInvisible` is true.
*/
getVisibleEqualsFilter(isVisible) {
if (typeof this.config.visible === 'boolean') {
return undefined;
}
if (!this.customPropertyFilters.visible) {
const property = this.propertyResolver.resolveVisiblePropertyPointer();
if (property) {
this.customPropertyFilters.visible = (0, query_1.propertyFilterBuilder)(property);
}
}
if (this.customPropertyFilters.visible) {
switch (this.customPropertyFilters.visible.schema.type) {
case 'formula':
return this.customPropertyFilters.visible.checkbox({
equals: isVisible,
});
default:
return this.customPropertyFilters.visible.equals(isVisible);
}
}
}
/**
* If `config.slug` is a property pointer, return a filter for `slugProperty = slug`.
* This filter is used by [[loadPageBySlug]] and possibly by [[getQueryParameters]].
*/
getSlugEqualsFilter(slug) {
if (!this.config.slug) {
return undefined;
}
if (!this.customPropertyFilters.slug) {
const property = this.propertyResolver.resolveSlugPropertyPointer();
if (property) {
this.customPropertyFilters.slug = (0, query_1.propertyFilterBuilder)(property);
}
}
if (this.customPropertyFilters.slug) {
switch (this.customPropertyFilters.slug.schema.type) {
case 'formula':
return this.customPropertyFilters.slug.string({
equals: slug,
});
default:
return this.customPropertyFilters.slug.equals(slug);
}
}
}
getDefaultSorts() {
return [this.sort.created_time.descending];
}
async buildCMSPage(args) {
var _a, _b;
const { page, children: content, reIndexChildren } = args;
const pageWithChildren = {
...page,
children: content,
};
this.rebuildIndexes(pageWithChildren, reIndexChildren);
const [slug, visible, title] = await Promise.all([
(_a = args.slug) !== null && _a !== void 0 ? _a : this.getSlug(pageWithChildren),
(_b = args.visible) !== null && _b !== void 0 ? _b : this.getVisible(pageWithChildren),
this.getTitle(pageWithChildren),
]);
const defaultFrontmatter = {
slug,
title,
visible,
};
const frontmatter = await this.config.getFrontmatter({
page: pageWithChildren,
defaultFrontmatter,
properties: (0, notion_api_1.getAllProperties)(page, this.schema),
}, this);
const finalFrontmatter = {
...frontmatter,
slug,
visible,
title,
};
DEBUG_CMS('build page %s: %o', page.id, finalFrontmatter);
const cmsPage = {
content: pageWithChildren,
frontmatter: finalFrontmatter,
};
this.pages.set(cmsPage.content.id, cmsPage);
return cmsPage;
}
async getTitle(page) {
if (this.config.title) {
const customTitle = await getCustomPropertyValue(this.config.title, page, this);
if (customTitle !== undefined) {
return customTitle;
}
}
return getPageTitle(page);
}
async getVisible(page) {
if (typeof this.config.visible === 'boolean') {
return this.config.visible;
}
const customVisible = await getCustomPropertyValue(this.config.visible, page, this);
if (typeof customVisible === 'object') {
// Formula return type
return Boolean((0, notion_api_1.getFormulaPropertyValueData)(customVisible));
}
return Boolean(customVisible);
}
async getSlug(page) {
if (this.config.slug) {
const customSlug = await getCustomPropertyValue(this.config.slug, page, this);
if (typeof customSlug === 'object' && 'type' in customSlug) {
// Formula return type
return String((0, notion_api_1.getFormulaPropertyValueData)(customSlug) || '');
}
return (0, notion_api_1.richTextAsPlainText)(customSlug) || defaultSlug(page);
}
return defaultSlug(page);
}
rebuildIndexes(page, reIndexChildren) {
this.notionObjects.addPage(page);
if (reIndexChildren) {
// Delete outdated data
this.backlinks.deleteBacklinksFromPage(page.id);
// Rebuild backlinks
(0, backlinks_1.buildBacklinks)([page], this.backlinks);
// Refresh object cache
(0, notion_api_1.visitChildBlocks)(page.children, (block) => this.notionObjects.addBlock(block, undefined));
}
}
}
exports.CMS = CMS;
////////////////////////////////////////////////////////////////////////////////
// Custom Properties
////////////////////////////////////////////////////////////////////////////////
/**
* Resolve [[CMSConfig]] options to property pointers.
* This is implemented as a separate class from [[CMS]] to improve type inference.
* See {@link CMS.propertyResolver}.
* @category CMS
*/
class CMSPropertyResolver {
constructor(cms) {
this.cms = cms;
this.config = cms.config;
}
/** If `config.slug` is a property pointer, returns it as a [[PropertyPointer]]. */
resolveSlugPropertyPointer() {
if (this.config.slug) {
return resolveCustomPropertyPointer(this.config.slug, this.cms);
}
}
/** If `config.visible` is a property pointer, returns it as a [[PropertyPointer]]. */
resolveVisiblePropertyPointer() {
if (typeof this.config.visible === 'boolean') {
return undefined;
}
return resolveCustomPropertyPointer(this.config.visible, this.cms);
}
resolveCustomPropertyPointer(customProperty) {
return resolveCustomPropertyPointer(customProperty, this.cms);
}
}
exports.CMSPropertyResolver = CMSPropertyResolver;
/**
* @category CMS
* @param page
* @returns The default slug for the page, based on the page's ID.
*/
function defaultSlug(page) {
return page.id.split('-').join('');
}
exports.defaultSlug = defaultSlug;
/**
* @category Page
* @category CMS
* @param page
* @returns {RichText} The title of `page`, as [[RichText]].
*/
function getPageTitle(page) {
const title = Object.values(page.properties).find((prop) => prop.type === 'title');
if (!title || title.type !== 'title') {
throw new Error(`Page does not have title property: ${page.id}`);
}
return title.title;
}
exports.getPageTitle = getPageTitle;
function resolveCustomPropertyPointer(customProperty, cms) {
if (typeof customProperty !== 'object') {
return cms.config.schema[customProperty];
}
if (customProperty.type === 'property') {
return customProperty.property;
}
return undefined;
}
/**
* Compute a custom property.
* @category CMS
* @param customProperty The custom property to compute.
* @param page
* @param cms
* @returns
*/
async function getCustomPropertyValue(customProperty, page, cms) {
if (typeof customProperty !== 'object') {
customProperty = {
type: 'property',
property: resolveCustomPropertyPointer(customProperty, cms),
};
}
switch (customProperty.type) {
case 'property':
return (0, notion_api_1.getPropertyValue)(page, customProperty.property);
case 'derived':
return customProperty.derive({ page }, cms);
default:
(0, util_1.unreachable)(customProperty);
}
}
exports.getCustomPropertyValue = getCustomPropertyValue;
////////////////////////////////////////////////////////////////////////////////
// Page Cache
////////////////////////////////////////////////////////////////////////////////
const DEBUG_CACHE = DEBUG_CMS.extend('cache');
class PageContentCache {
constructor(config = {}) {
this.config = config;
this.cache = new Map();
this.setup = false;
}
get directory() {
return this.config.directory;
}
get maxPageContentAgeMs() {
var _a;
return (_a = this.config.maxPageContentAgeMs) !== null && _a !== void 0 ? _a : Infinity;
}
get minPageContentAgeMs() {
var _a;
return (_a = this.config.minPageContentAgeMs) !== null && _a !== void 0 ? _a : 0;
}
async getPageContent(notion, pageIdOrPage) {
var _a;
const pageId = typeof pageIdOrPage === 'string' ? pageIdOrPage : pageIdOrPage.id;
let newPage = typeof pageIdOrPage === 'object' ? pageIdOrPage : undefined;
const { cached, fromMemory } = await this.getCacheContents(pageId);
if (cached) {
const cacheAgeMs = Date.now() - cached.fetchedAtTs;
if (cacheAgeMs < this.minPageContentAgeMs) {
DEBUG_CACHE('%s hit (%s): age %s < %s', pageId, fromMemory ? 'memory' : 'disk', cacheAgeMs, this.minPageContentAgeMs);
return {
children: cached.children,
hit: fromMemory,
};
}
if (cacheAgeMs < this.maxPageContentAgeMs) {
// Check last modified time
newPage !== null && newPage !== void 0 ? newPage : (newPage = await this.fetchPage(notion, pageId));
if (newPage && newPage.last_edited_time === cached.last_edited_at) {
DEBUG_CACHE('%s hit (%s): last_edited_time same %s', pageId, fromMemory ? 'memory' : 'disk', newPage.last_edited_time);
return {
children: cached.children,
hit: fromMemory,
};
}
}
}
newPage !== null && newPage !== void 0 ? newPage : (newPage = await this.fetchPage(notion, pageId));
DEBUG_CACHE('%s miss', pageId);
// Even if we didn't get a whole page, we can still fetch the children
// and the last_edited_time
const content = await (0, notion_api_1.getChildBlocksWithChildrenRecursively)(notion, pageId);
const now = new Date();
const cacheEntry = {
fetchedAtTs: now.getTime(),
children: content,
last_edited_at: (_a = newPage === null || newPage === void 0 ? void 0 : newPage.last_edited_time) !== null && _a !== void 0 ? _a : now.toString(),
};
await this.storeCacheContents(pageId, cacheEntry);
const result = {
children: content,
hit: false,
page: newPage,
};
return result;
}
async setupDirectory() {
if (this.setup === false && this.directory) {
await fs.mkdir(this.directory, { recursive: true });
this.setup = true;
}
}
async getCacheContents(pageId) {
let cached = this.cache.get(pageId);
const fromMemory = Boolean(cached);
const cacheFileName = this.getPageCacheFileName(pageId);
if (!cached && cacheFileName) {
try {
cached = JSON.parse(await fs.readFile(cacheFileName, 'utf8'));
}
catch (error) {
if (error.code !== 'ENOENT') {
throw error;
}
}
}
return { cached, fromMemory };
}
async storeCacheContents(pageId, cacheEntry) {
const cacheFileName = this.getPageCacheFileName(pageId);
this.cache.set(pageId, cacheEntry);
if (cacheFileName) {
try {
await this.setupDirectory();
// TODO: implement atomic write as write then move
await fs.writeFile(path.join(cacheFileName), JSON.stringify(cacheEntry));
}
catch (error) {
console.warn('Failed to write cache file', error);
}
}
}
async fetchPage(notion, pageId) {
const page = await notion.pages.retrieve({
page_id: pageId,
});
if ('last_edited_time' in page) {
return page;
}
}
getPageCacheFileName(pageId) {
if (!this.directory) {
return;
}
return path.join(this.directory, `cache.pageContent.${pageId}.json`);
}
}
////////////////////////////////////////////////////////////////////////////////
// Asset cache
////////////////////////////////////////////////////////////////////////////////
const DEBUG_ASSETS = DEBUG_CMS.extend('assets');
class AssetCache {
constructor(config) {
this.config = config;
this.assetRequestCache = new Map();
this.assetFileCache = new Map();
this.setup = false;
}
get directory() {
return this.config.directory;
}
async setupDirectory() {
if (this.setup === false && this.directory) {
await fs.mkdir(this.directory, { recursive: true });
this.setup = true;
}
}
/** Get an asset that was loading into memory by this process already. */
async getCachedAsset(request) {
const assetRequestKey = (0, assets_1.getAssetRequestKey)(request);
const asset = this.assetRequestCache.get(assetRequestKey);
if (!asset) {
return;
}
const assetKey = (0, assets_1.getAssetKey)(asset);
const [path, hit] = await (0, cache_1.getFromCache)('fill', () => this.assetFileCache.get(assetKey), () => (0, assets_1.ensureAssetInDirectory)({
asset,
directory: this.directory,
cacheBehavior: 'read-only',
emojiSourceDirectory: this.config.emojiSourceDirectory,
}));
if (path) {
(0, cache_1.fillCache)('fill', hit, () => this.assetFileCache.set(assetKey, path));
}
return path;
}
/** Resolve an asset request to an asset using the in-memory cache */
async performAssetRequest(args) {
const { cacheBehavior, request } = args;
const assetRequestKey = (0, assets_1.getAssetRequestKey)(request);
const [asset, assetHit] = await (0, cache_1.getFromCache)(cacheBehavior, () => this.assetRequestCache.get(assetRequestKey), () => (0, assets_1.performAssetRequest)(args));
if (!asset) {
DEBUG_ASSETS('asset request not found: %s', assetRequestKey);
return;
}
(0, cache_1.fillCache)(cacheBehavior, assetHit, () => this.assetRequestCache.set(assetRequestKey, asset));
return asset;
}
/** Download the given `asset` to disk, and add it to the in-memory cache key for `request`. */
async downloadAsset(args) {
const { cacheBehavior, request, asset } = args;
const assetRequestKey = (0, assets_1.getAssetRequestKey)(request);
if (asset.type === 'external' && !this.config.downloadExternalAssets) {
return asset.external.url;
}
await this.setupDirectory();
const assetKey = (0, assets_1.getAssetKey)(asset);
const [assetFileName, assetFileHit] = await (0, cache_1.getFromCache)(cacheBehavior, () => this.assetFileCache.get(assetKey), () => (0, assets_1.ensureAssetInDirectory)({
asset,
directory: this.directory,
emojiSourceDirectory: this.config.emojiSourceDirectory,
}));
if (!assetFileName) {
DEBUG_ASSETS('asset not found: %s', assetRequestKey);
return;
}
(0, cache_1.fillCache)(cacheBehavior, assetFileHit, () => this.assetFileCache.set(assetKey, assetFileName));
return assetFileName;
}
/** Download the given `request` to disk, and fill related in-memory caches, if needed. */
async downloadAssetRequest(args) {
const { cacheBehavior, request } = args;
const assetRequestKey = (0, assets_1.getAssetRequestKey)(request);
const asset = await this.performAssetRequest(args);
if (!asset) {
return;
}
try {
return await this.downloadAsset({
asset,
...args,
});
}
catch (error) {
if (error instanceof Error &&
error.name === assets_1.DOWNLOAD_PERMISSION_ERROR &&
asset.type === 'file' &&
!cacheBehavior) {
DEBUG_ASSETS('asset expired: %s', assetRequestKey);
return this.downloadAssetRequest({
...args,
cacheBehavior: 'refresh',
});
}
throw error;
}
}
}
function examples() {
const notion = undefined;
const myProps = (page) => ({});
const cms = new CMS({
notion,
database_id: 'example',
slug: 'productCode',
visible: 'publicAccess',
schema: (0, notion_api_1.inferDatabaseSchema)({
productCode: {
name: 'Product Code',
type: 'rich_text',
},
Subtitle: {
type: 'rich_text',
},
publicAccess: {
name: 'Public Access',
type: 'checkbox',
},
Date: {
type: 'date',
},
}),
getFrontmatter: ({ properties }) => ({
...properties,
productCode: (0, notion_api_1.richTextAsPlainText)(properties.productCode),
}),
});
cms.query({
filter: cms.filter.and(cms.filter.Date.before('2020-01-01'), cms.filter.publicAccess.equals(true)),
sorts: [cms.sort.last_edited_time.descending],
});
const drafts = cms.scope({
filter: cms.getVisibleEqualsFilter(false),
showInvisible: true,
});
}
//# sourceMappingURL=content-management-system.js.map