@vrerv/md-to-notion
Version:
An upload of markdown files to a hierarchy of Notion pages.
331 lines • 14.3 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.archivePage = exports.archiveChildPages = exports.syncToNotion = exports.collectCurrentFiles = exports.findMaxDepth = void 0;
const logging_1 = require("./logging");
const merge_blocks_1 = require("./merge-blocks");
function isBlockObjectResponse(child) {
return child.object === "block";
}
const logger = (0, logging_1.makeConsoleLogger)("sync-to-notion");
const NOTION_BLOCK_LIMIT = 100;
/**
* Generates a unique key for a page based on its parent ID and title to find markdown file in Notion.
* @param parentId - The notion page ID of the parent page.
* @param title
*/
function commonPageKey(parentId, title) {
return `${parentId}/${title}`;
}
function getPageTitle(pageResponse) {
var _a, _b, _c;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const properties = Object.values(pageResponse.properties || {})
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
.find((prop) => prop.type === "title");
if (properties && properties.title && properties.title.length > 0) {
const title = ((_a = properties.title[0]) === null || _a === void 0 ? void 0 : _a.plain_text) || ((_c = (_b = properties.title[0]) === null || _b === void 0 ? void 0 : _b.text) === null || _c === void 0 ? void 0 : _c.content);
if (title) {
return title;
}
}
logger(logging_1.LogLevel.ERROR, "Error no title found", { pageResponse });
throw new Error(
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
`No title found. Please set a title for the page ${pageResponse.url} and try again.`);
}
function newNotionPageLink(response) {
return {
id: response.id,
link: response.url,
};
}
/**
* find the maximum depth of the req block
* @param block
* @param depth
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const findMaxDepth = (block, depth = 0) => {
if (!block || !block.children) {
return depth;
}
let maxDepth = depth;
for (const child of block.children) {
const childNode = child[child.type];
const childDepth = (0, exports.findMaxDepth)(childNode, depth + 1);
maxDepth = Math.max(maxDepth, childDepth);
}
return maxDepth;
};
exports.findMaxDepth = findMaxDepth;
async function collectCurrentFiles(notion, rootPageId) {
const linkMap = new Map();
async function collectPages(pageId, parentTitle) {
const response = await notion.pages.retrieve({
page_id: pageId,
});
logger(logging_1.LogLevel.DEBUG, "", response);
const pageTitle = getPageTitle(response);
if (response.object === "page") {
linkMap.set(commonPageKey(parentTitle, pageTitle), newNotionPageLink(response));
const childrenResponse = await notion.blocks.children.list({
block_id: pageId,
});
for (const child of childrenResponse.results) {
if (isBlockObjectResponse(child) && child.type === "child_page") {
await collectPages(child.id, pageId === rootPageId ? "." : parentTitle + "/" + pageTitle);
}
}
}
}
await collectPages(rootPageId, ".");
return linkMap;
}
exports.collectCurrentFiles = collectCurrentFiles;
/**
* Synchronizes a folder structure to a Notion page.
*
* @param notion
* @param pageId - The ID of the Notion page to sync the content to.
* @param dir - The folder structure to sync.
* @param linkMap
* @param deleteNonExistentFiles - Whether to delete pages in Notion that don't exist locally
* @returns A promise that resolves when the synchronization is complete.
*/
async function syncToNotion(notion, pageId, dir, linkMap = new Map(), deleteNonExistentFiles = false) {
async function appendBlocksInChunks(pageId,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
blocks, afterId = null) {
const limitChild = (0, exports.findMaxDepth)({ children: blocks }, 0) > 3;
// Append blocks in chunks of NOTION_BLOCK_LIMIT
for (let i = 0; i < blocks.length; i += NOTION_BLOCK_LIMIT) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const children = {};
const chunk = blocks
.slice(i, i + NOTION_BLOCK_LIMIT)
.map((block, index) => {
var _a, _b, _c;
if (limitChild && ((_a = block[block.type]) === null || _a === void 0 ? void 0 : _a.children)) {
children[index] = (_b = block[block.type]) === null || _b === void 0 ? void 0 : _b.children;
(_c = block[block.type]) === null || _c === void 0 ? true : delete _c.children;
}
return block;
});
try {
const response = await notion.blocks.children.append({
block_id: pageId,
children: chunk,
after: afterId ? afterId : undefined,
});
// Check for children in the chunk and append them separately
for (const index in children) {
if (children[index]) {
await appendBlocksInChunks(
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
response.results[index].id,
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
children[index]);
}
}
}
catch (error) {
logger(logging_1.LogLevel.ERROR, "Error appending blocks", { error, chunk });
throw error;
}
}
}
async function createOrUpdatePage(folderName, parentId, parentName, onUpdated) {
const key = commonPageKey(parentName, folderName);
logger(logging_1.LogLevel.INFO, "Create page", { key: key });
if (linkMap.has(key)) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const pageId = linkMap.get(key).id;
await onUpdated(pageId);
return pageId;
}
else {
const response = await notion.pages.create({
parent: { page_id: parentId },
properties: {
title: [{ text: { content: folderName } }],
},
});
linkMap.set(key, newNotionPageLink(response));
return response.id;
}
}
async function getExistingBlocks(notion, pageId) {
const existingBlocks = [];
let cursor = undefined;
do {
const response = await notion.blocks.children.list({
block_id: pageId,
start_cursor: cursor,
});
existingBlocks.push(...response.results.filter(isBlockObjectResponse));
cursor = response.has_more ? response.next_cursor : undefined;
} while (cursor);
// you have to get children blocks if the block has children
for (const block of existingBlocks) {
if (block.has_children) {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
block[block.type].children = await getExistingBlocks(notion, block.id);
}
}
return existingBlocks;
}
async function syncFolder(folder, parentId, parentName, createFolder = true, pages, folderPageIds) {
let folderPageId = parentId;
if (createFolder) {
folderPageId = await createOrUpdatePage(folder.name, parentId, parentName, async (_) => {
/* do nothing */
});
}
folderPageIds.add(folderPageId);
const childParentName = dir.name === folder.name ? parentName : parentName + "/" + folder.name;
for (const file of folder.files) {
const pageId = await createOrUpdatePage(file.fileName, folderPageId, childParentName, async (_) => {
/* do nothing */
});
pages.push({ pageId: pageId, file: file });
}
for (const subfolder of folder.subfolders) {
await syncFolder(subfolder, folderPageId, childParentName, true, pages, folderPageIds);
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
async function updateBlocks(pageId, newBlocks) {
const blockIdSetToDelete = new Set();
const existingBlocks = await getExistingBlocks(notion, pageId);
await (0, merge_blocks_1.mergeBlocks)(existingBlocks, newBlocks,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
async (blocks, after) => {
var _a;
let afterId = after === null || after === void 0 ? void 0 : after.id;
let appendBlocks = blocks;
if ((after === null || after === undefined) &&
existingBlocks.length > 0 &&
((_a = existingBlocks[0]) === null || _a === void 0 ? void 0 : _a.id)) {
// to overcome the limitation of the Notion API that requires an after block to append blocks,
// append to after first block and delete first block
const firstBlock = existingBlocks[0];
afterId = firstBlock.id;
appendBlocks = blockIdSetToDelete.has(afterId)
? blocks
: [
...blocks,
{
...firstBlock,
/* to create a new block or avoid any interference with existing blocks, set id undefined */
id: undefined,
},
];
blockIdSetToDelete.add(afterId);
}
logger(logging_1.LogLevel.INFO, "Appending blocks", { appendBlocks, after });
await appendBlocksInChunks(pageId, appendBlocks, afterId);
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
async (block) => {
blockIdSetToDelete.add(block.id);
});
for (const blockId of blockIdSetToDelete) {
logger(logging_1.LogLevel.INFO, "Deleting a block", { blockId });
await notion.blocks.delete({ block_id: blockId });
}
}
const pages = [];
const folderPageIds = new Set();
await syncFolder(dir, pageId, ".", false, pages, folderPageIds);
const linkUrlMap = new Map(Array.from(linkMap.entries()).map(([key, value]) => [key, value.link]));
for (const page of pages) {
const blocks = page.file.getContent(linkUrlMap);
logger(logging_1.LogLevel.INFO, "Update blocks", {
pageId: page.pageId,
file: page.file,
newBlockSize: blocks.length,
});
await updateBlocks(page.pageId, blocks);
}
// Track which pages from Notion were found in the local directory
// Include both file pages and their parent folder pages
const processedNotionPageIds = new Set([
...pages.map(page => page.pageId),
...folderPageIds,
]);
if (deleteNonExistentFiles) {
// Track pages that we've archived in this run
const archivedPages = new Set();
// Sort keys by path length so that we delete parent paths first
// This reduces API calls because archiving a parent will archive all children
const sortedEntries = Array.from(linkMap.entries()).sort((a, b) => a[0].length - b[0].length);
for (const [key, pageLink] of sortedEntries) {
const isRootPage = pageLink.id.replace(/-/g, "") === pageId.replace(/-/g, "");
if (isRootPage) {
continue;
}
if (processedNotionPageIds.has(pageLink.id)) {
continue;
}
// Check if any ancestor path has been archived
// For a path like "./1/2/3/file", check if "./1", "./1/2", or "./1/2/3" is archived
let hasArchivedAncestor = false;
const pathParts = key.split("/");
// Build paths from root to the current path and check each
if (pathParts.length > 1) {
for (let i = 1; i < pathParts.length; i++) {
const ancestorPath = pathParts.slice(0, i).join("/");
if (archivedPages.has(ancestorPath)) {
logger(logging_1.LogLevel.INFO, `Skipping page with archived ancestor: ${key}`, {
ancestorPath,
pageId: pageLink.id,
});
hasArchivedAncestor = true;
break;
}
}
}
if (hasArchivedAncestor) {
continue;
}
try {
logger(logging_1.LogLevel.INFO, `Deleting page: ${key}`);
await archivePage(notion, pageLink.id);
archivedPages.add(key);
}
catch (error) {
logger(logging_1.LogLevel.ERROR, `Error deleting page: ${key}`, {
error,
pageId: pageLink.id,
});
throw error;
}
}
}
}
exports.syncToNotion = syncToNotion;
async function archiveChildPages(notion, pageId) {
logger(logging_1.LogLevel.INFO, `Archiving child pages of: ${pageId}`);
const childrenResponse = await notion.blocks.children.list({
block_id: pageId,
});
for (const child of childrenResponse.results) {
if (isBlockObjectResponse(child) && child.type === "child_page") {
await archivePage(notion, child.id);
}
}
}
exports.archiveChildPages = archiveChildPages;
async function archivePage(notion, pageId) {
await notion.pages.update({
page_id: pageId,
archived: true,
});
}
exports.archivePage = archivePage;
//# sourceMappingURL=sync-to-notion.js.map