UNPKG

astro-loader-bluesky-posts

Version:

Astro loader for loading Bluesky posts and threads using post URLs or AT-URIs.

761 lines (757 loc) 24.9 kB
import { RichText, AtpAgent } from '@atproto/api'; import { isNotFoundPost, isBlockedPost, isThreadViewPost } from '@atproto/api/dist/client/types/app/bsky/feed/defs.js'; import { z } from 'astro/zod'; // src/index.ts var defaultConfig = { urlTextType: "post-text", newlineHandling: "none", fetchThread: false, threadDepth: 1, threadParentHeight: 1, fetchOnlyAuthorReplies: false }; var BlueskyPostsLoaderConfigSchema = z.object({ /** * List of Bluesky post URLs or {@link https://atproto.com/specs/at-uri-scheme AT-URIs}. */ uris: z.array(z.string()), /** * The type of text to display for links when generating renderable HTML: * - `'domain-path'`: Shows the link's domain and path. * - `'post-text'`: Uses the link text as shown in the post. * * @default 'post-text' */ linkTextType: z.enum(["domain-path", "post-text"]).default(defaultConfig.urlTextType), /** * The way for processing `\n` when generating renderable HTML: * - `'none'`: Keep as is. * - `'break'`: Replace consecutive `\n` with `<br>`. * - `'paragraph'`: Wrap paragraphs with `<p>` while removing standalone `\n`. * * @default 'none' */ newlineHandling: z.enum(["none", "break", "paragraph"]).default(defaultConfig.newlineHandling), /** * Whether to fetch the post's thread including replies and parents. * * @default false */ fetchThread: z.boolean().default(defaultConfig.fetchThread), /** * The depth of the descendant post tree to fetch if fetching the thread. * Specifies how many levels of reply depth should be included. * * @default 1 */ threadDepth: z.number().min(0).max(1e3).default(defaultConfig.threadDepth), /** * The height of the ancestor post tree to fetch if fetching the thread. * Specifies how many levels of parent posts should be included. * * @default 1 */ threadParentHeight: z.number().min(0).max(1e3).default(defaultConfig.threadParentHeight), /** * Fetches only the post author's replies at the specified `threadDepth`. * * - `false` (default): Includes all replies without filtering. * - `true`: Returns only the author's replies as a flat array, ignoring `threadParentHeight`, * with no `parent` field. * * @default false */ fetchOnlyAuthorReplies: z.boolean().default(defaultConfig.fetchOnlyAuthorReplies) }); var UnknownSchema = z.object({ $type: z.string() }).passthrough(); var labelSchema = z.object({ $type: z.literal("com.atproto.label.defs#label").optional(), ver: z.number().optional(), src: z.string(), uri: z.string(), cid: z.string().optional(), val: z.string(), neg: z.boolean().optional(), cts: z.string(), exp: z.string().optional(), sig: z.instanceof(Uint8Array).optional() }); var listPurposeSchema = z.union([ z.literal("app.bsky.graph.defs#modlist"), z.literal("app.bsky.graph.defs#curatelist"), z.literal("app.bsky.graph.defs#referencelist"), z.string().and(z.object({})) ]); var listViewerStateSchema = z.object({ $type: z.literal("app.bsky.graph.defs#listViewerState").optional(), muted: z.boolean().optional(), blocked: z.string().optional() }); var listViewBasicSchema = z.object({ $type: z.literal("app.bsky.graph.defs#listViewBasic").optional(), uri: z.string(), cid: z.string(), name: z.string(), purpose: listPurposeSchema, avatar: z.string().optional(), listItemCount: z.number().optional(), labels: z.array(labelSchema).optional(), viewer: listViewerStateSchema.optional(), indexedAt: z.string().optional() }); var profileAssociatedChatSchema = z.object({ $type: z.literal("app.bsky.graph.defs#profileAssociatedChat").optional(), allowIncoming: z.union([ z.literal("all"), z.literal("none"), z.literal("following"), z.string().and(z.object({})) ]) }); var profileAssociatedSchema = z.object({ $type: z.literal("app.bsky.actor.defs#profileAssociated").optional(), lists: z.number().optional(), feedgens: z.number().optional(), starterPacks: z.number().optional(), labeler: z.boolean().optional(), chat: profileAssociatedChatSchema.optional() }); var knownFollowersSchema = z.object({ $type: z.literal("app.bsky.actor.defs#knownFollowers").optional(), count: z.number(), followers: z.lazy(() => profileViewBasicSchema.array()) }); var viewerStateSchema = z.object({ $type: z.literal("app.bsky.actor.defs#viewerState").optional(), muted: z.boolean().optional(), mutedByList: listViewBasicSchema.optional(), blockedBy: z.boolean().optional(), blocking: z.string().optional(), blockingByList: listViewBasicSchema.optional(), following: z.string().optional(), followedBy: z.string().optional(), knownFollowers: knownFollowersSchema.optional() }); var profileViewBasicSchema = z.object({ $type: z.literal("app.bsky.actor.defs#profileViewBasic").optional(), did: z.string(), handle: z.string(), displayName: z.string().optional(), avatar: z.string().optional(), associated: profileAssociatedSchema.optional(), viewer: viewerStateSchema.optional(), labels: z.array(labelSchema).optional(), createdAt: z.string().optional() }); var aspectRatioSchema = z.object({ $type: z.literal("app.bsky.embed.defs#aspectRatio").optional(), width: z.number(), height: z.number() }); var viewImageSchema = z.object({ $type: z.literal("app.bsky.embed.images#viewImage").optional(), thumb: z.string(), fullsize: z.string(), alt: z.string(), aspectRatio: aspectRatioSchema.optional() }); var embedImagesViewSchema = z.object({ $type: z.literal("app.bsky.embed.images#view").optional(), images: z.array(viewImageSchema) }); var embedVideoViewSchema = z.object({ $type: z.literal("app.bsky.embed.video#view").optional(), cid: z.string(), playlist: z.string(), thumbnail: z.string().optional(), alt: z.string().optional(), aspectRatio: aspectRatioSchema.optional() }); var viewExternalSchema = z.object({ $type: z.literal("app.bsky.embed.external#viewExternal").optional(), uri: z.string(), title: z.string(), description: z.string(), thumb: z.string().optional() }); var embedExternalViewSchema = z.object({ $type: z.literal("app.bsky.embed.external#view").optional(), external: viewExternalSchema }); var profileViewSchema = z.object({ $type: z.literal("app.bsky.actor.defs#profileView").optional(), did: z.string(), handle: z.string(), displayName: z.string().optional(), description: z.string().optional(), avatar: z.string().optional(), associated: profileAssociatedSchema.optional(), indexedAt: z.string().optional(), createdAt: z.string().optional(), // viewer: viewerStateSchema.optional(), labels: z.array(labelSchema).optional() }); var byteSliceSchema = z.object({ $type: z.literal("app.bsky.richtext.facet#byteSlice").optional(), byteStart: z.number(), byteEnd: z.number() }); var mentionSchema = z.object({ $type: z.literal("app.bsky.richtext.facet#mention").optional(), did: z.string() }); var linkSchema = z.object({ $type: z.literal("app.bsky.richtext.facet#link").optional(), uri: z.string() }); var tagSchema = z.object({ $type: z.literal("app.bsky.richtext.facet#tag").optional(), tag: z.string() }); var richTextMainSchema = z.object({ $type: z.literal("app.bsky.richtext.facet").optional(), index: byteSliceSchema, features: z.array(z.union([mentionSchema, linkSchema, tagSchema])) }); var generatorViewerStateSchema = z.object({ $type: z.literal("app.bsky.feed.defs#generatorViewerState").optional(), like: z.string().optional() }); var generatorViewSchema = z.object({ $type: z.literal("app.bsky.feed.defs#generatorView").optional(), uri: z.string(), cid: z.string(), did: z.string(), creator: profileViewSchema, displayName: z.string(), description: z.string().optional(), descriptionFacets: z.array(richTextMainSchema).optional(), avatar: z.string().optional(), likeCount: z.number().optional(), acceptsInteractions: z.boolean().optional(), labels: z.array(labelSchema).optional(), viewer: generatorViewerStateSchema.optional(), contentMode: z.union([ z.literal("app.bsky.feed.defs#contentModeUnspecified"), z.literal("app.bsky.feed.defs#contentModeVideo"), z.string().and(z.object({})) ]).optional(), indexedAt: z.string() }); var viewRecordSchema = z.object({ $type: z.literal("app.bsky.embed.record#viewRecord").optional(), uri: z.string(), cid: z.string(), author: profileViewBasicSchema, value: z.record(z.unknown()), labels: z.array(labelSchema).optional(), replyCount: z.number().optional(), repostCount: z.number().optional(), likeCount: z.number().optional(), quoteCount: z.number().optional(), embeds: z.array( z.union([ embedImagesViewSchema, embedVideoViewSchema, embedExternalViewSchema, z.lazy(() => embedRecordViewSchema), z.lazy(() => embedRecordWithMediaViewSchema) ]) ).optional(), indexedAt: z.string() }); var viewNotFoundSchema = z.object({ $type: z.literal("app.bsky.embed.record#viewNotFound").optional(), uri: z.string(), notFound: z.literal(true) }); var blockedAuthorSchema = z.object({ $type: z.literal("app.bsky.feed.defs#blockedAuthor").optional(), did: z.string(), viewer: viewerStateSchema.optional() }); var viewBlockedSchema = z.object({ $type: z.literal("app.bsky.embed.record#viewBlocked").optional(), uri: z.string(), blocked: z.literal(true), author: blockedAuthorSchema }); var viewDetachedSchema = z.object({ $type: z.literal("app.bsky.embed.record#viewDetached").optional(), uri: z.string(), detached: z.literal(true) }); var listViewSchema = z.object({ $type: z.literal("app.bsky.graph.defs#listView").optional(), uri: z.string(), cid: z.string(), creator: profileViewSchema, name: z.string(), purpose: listPurposeSchema, description: z.string().optional(), descriptionFacets: z.array(richTextMainSchema).optional(), avatar: z.string().optional(), listItemCount: z.number().optional(), labels: z.array(labelSchema).optional(), viewer: listViewerStateSchema.optional(), indexedAt: z.string() }); var labelerViewerStateSchema = z.object({ $type: z.literal("app.bsky.labeler.defs#labelerViewerState").optional(), like: z.string().optional() }); var labelerViewSchema = z.object({ $type: z.literal("app.bsky.labeler.defs#labelerView").optional(), uri: z.string(), cid: z.string(), creator: profileViewSchema, likeCount: z.number().optional(), viewer: labelerViewerStateSchema.optional(), indexedAt: z.string(), labels: z.array(labelSchema).optional() }); var starterPackViewBasicSchema = z.object({ $type: z.literal("app.bsky.graph.defs#starterPackViewBasic").optional(), uri: z.string(), cid: z.string(), record: z.record(z.unknown()), creator: profileViewBasicSchema, listItemCount: z.number().optional(), joinedWeekCount: z.number().optional(), joinedAllTimeCount: z.number().optional(), labels: z.array(labelSchema).optional(), indexedAt: z.string() }); var embedRecordViewSchema = z.object({ $type: z.literal("app.bsky.embed.record#view").optional(), record: z.union([ viewRecordSchema, viewNotFoundSchema, viewBlockedSchema, viewDetachedSchema, generatorViewSchema, listViewSchema, labelerViewSchema, starterPackViewBasicSchema ]) }); var embedRecordWithMediaViewSchema = z.object({ $type: z.literal("app.bsky.embed.recordWithMedia#view").optional(), record: embedRecordViewSchema, media: z.union([ embedImagesViewSchema, embedVideoViewSchema, embedExternalViewSchema ]) }); var strongRefMainSchema = z.object({ $type: z.literal("com.atproto.repo.strongRef").optional(), uri: z.string(), cid: z.string() }); var replyRefSchema = z.object({ $type: z.literal("app.bsky.feed.post#replyRef").optional(), root: strongRefMainSchema, parent: strongRefMainSchema }); var selfLabelSchema = z.object({ $type: z.literal("com.atproto.label.defs#selfLabel").optional(), val: z.string() }); var selfLabelsSchema = z.object({ $type: z.literal("com.atproto.label.defs#selfLabels").optional(), values: z.array(selfLabelSchema) }); var recordEmbedImage = z.object({ $type: z.literal("app.bsky.embed.images"), images: z.array( z.object({ alt: z.string().optional(), aspectRatio: z.object({ height: z.number(), width: z.number() }).optional(), image: z.object({ $type: z.literal("blob"), ref: z.object({ $link: z.string() }), mimeType: z.string(), size: z.number() }) }) ) }); var recordEmbedVideo = z.object({ $type: z.literal("app.bsky.embed.video"), alt: z.string().optional(), aspectRatio: z.object({ height: z.number(), width: z.number() }).optional(), video: z.object({ $type: z.literal("blob"), ref: z.object({ $link: z.string() }), mimeType: z.string(), size: z.number() }) }); var recordEmbedExternal = z.object({ $type: z.literal("app.bsky.embed.external"), external: z.object({ uri: z.string().url(), title: z.string().optional(), description: z.string().optional(), thumb: z.object({ $type: z.literal("blob"), ref: z.object({ $link: z.string() }), mimeType: z.string(), size: z.number() }).optional() }) }); var recordEmbedRecord = z.object({ $type: z.literal("app.bsky.embed.record"), record: z.object({ uri: z.string(), cid: z.string() }) }); var EmbedRecordWithMediaSchema = z.object({ $type: z.literal("app.bsky.embed.recordWithMedia"), record: recordEmbedRecord, media: z.union([ recordEmbedImage, recordEmbedVideo, recordEmbedExternal, recordEmbedRecord, UnknownSchema ]) }); var recordSchema = z.object({ $type: z.literal("app.bsky.feed.post"), text: z.string(), facets: z.array(richTextMainSchema).optional(), reply: replyRefSchema.optional(), embed: z.union([ recordEmbedImage, recordEmbedVideo, recordEmbedExternal, recordEmbedRecord, EmbedRecordWithMediaSchema, UnknownSchema ]).optional(), langs: z.array(z.string()).optional(), labels: selfLabelsSchema.optional(), tags: z.array(z.string()).optional(), createdAt: z.string() }).catchall(z.unknown()); var threadgateViewSchema = z.object({ $type: z.literal("app.bsky.feed.defs#threadgateView").optional(), uri: z.string().optional(), cid: z.string().optional(), record: z.record(z.unknown()).optional(), lists: z.array(listViewBasicSchema).optional() }); var postViewSchema = z.object({ $type: z.literal("app.bsky.feed.defs#postView").optional(), uri: z.string(), cid: z.string(), author: profileViewBasicSchema, record: recordSchema, embed: z.union([ embedImagesViewSchema, embedVideoViewSchema, embedExternalViewSchema, embedRecordViewSchema, embedRecordWithMediaViewSchema, UnknownSchema ]).optional(), replyCount: z.number().optional(), repostCount: z.number().optional(), likeCount: z.number().optional(), quoteCount: z.number().optional(), indexedAt: z.string(), viewer: viewerStateSchema.optional(), label: z.array(labelSchema).optional(), threadgate: threadgateViewSchema.optional() }); var postViewExtendedSchema = postViewSchema.and( z.object({ link: z.string(), html: z.string() }) ); var notFoundPostSchema = z.object({ $type: z.literal("app.bsky.feed.defs#notFoundPost").optional(), uri: z.string(), notFound: z.literal(true) }); var blockedPostSchema = z.object({ $type: z.literal("app.bsky.feed.defs#blockedPost").optional(), uri: z.string(), blocked: z.literal(true), author: blockedAuthorSchema }); var postWithThreadViewSchema = z.object({ uri: z.string(), post: postViewExtendedSchema, parent: z.union([ z.lazy(() => threadViewPostSchema), notFoundPostSchema, blockedPostSchema, UnknownSchema ]).optional(), replies: z.array( z.union([ z.lazy(() => threadViewPostSchema), notFoundPostSchema, blockedPostSchema, UnknownSchema ]) ).optional() }); var postWithThreadViewExtendedSchema = postWithThreadViewSchema.extend({ replies: z.union([ z.array( z.union([ z.lazy(() => threadViewPostSchema), notFoundPostSchema, blockedPostSchema, UnknownSchema ]) ), z.array(postViewExtendedSchema) ]).optional() }); var threadViewPostSchema = postWithThreadViewSchema.omit({ uri: true }); async function getAtUri(uri, agent) { try { const parts = uri.startsWith("at:") ? uri.split("/") : new URL(uri).pathname.split("/"); let id = parts[2]; const postId = parts[4]; if (!id.startsWith("did:")) { const handleResolution = await agent.resolveHandle({ handle: id }); id = handleResolution.data.did; } return `at://${id}/app.bsky.feed.post/${postId}`; } catch { throw new Error(`Invalid post URL: ${uri}.`); } } function escapeHTML(str) { const escapeMap = { "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }; return str?.replace(/[&<>"']/g, (match) => escapeMap[match] || match) ?? ""; } function getDomainAndPath(url) { if (url) { const parsedUrl = new URL(url); return `${parsedUrl.hostname}${parsedUrl.pathname}`; } return ""; } function renderPostAsHtml(richText, options) { const { linkTextType, newlineHandling } = options; const rt = new RichText(richText); let html = ""; for (const segment of rt.segments()) { if (segment.isLink()) { html += `<a href="${escapeHTML(segment.link?.uri)}">${escapeHTML(linkTextType === "post-text" ? segment.text : getDomainAndPath(segment.link?.uri))}</a>`; } else if (segment.isMention()) { html += `<a href="https://bsky.app/profile/${escapeHTML(segment.mention?.did)}">${escapeHTML(segment.text)}</a>`; } else if (segment.isTag()) { html += `<a href="https://bsky.app/hastag/${escapeHTML(segment.tag?.tag)}">#${escapeHTML(segment.tag?.tag)}</a>`; } else { html += escapeHTML(segment.text); } } if (newlineHandling !== "none") { if (newlineHandling === "break") { html = html.replace(/\n+/g, "<br/ >\n"); } else if (newlineHandling === "paragraph") { html = html.split("\n").map((line) => { const l = line.trim(); if (l.length > 0) return `<p>${l}</p>`; }).join(""); } } return html; } function atUriToPostUri(atUri) { const [, , did, , postId] = atUri.split("/"); return `https://bsky.app/profile/${did}/post/${postId}`; } function getOnlyAuthorReplies(replies, depth, authorDid) { if (!replies || replies.length === 0 || depth <= 0) return []; const filtered = replies.filter( (item) => isThreadViewPost(item) && item.post.author.did === authorDid ); return filtered.reduce((acc, item) => { acc.push(item.post); const childPosts = getOnlyAuthorReplies(item.replies, depth - 1, authorDid); acc.push(...childPosts); return acc; }, []); } // src/index.ts function blueskyPostsLoader(userConfig) { return { name: "astro-loader-bluesky-posts", schema: userConfig.fetchThread ? postWithThreadViewExtendedSchema : postViewExtendedSchema, async load({ logger, store, parseData, generateDigest, meta }) { const parsedConfig = BlueskyPostsLoaderConfigSchema.safeParse(userConfig); if (!parsedConfig.success) { logger.error( `The configuration provided is invalid. ${parsedConfig.error.issues.map((issue) => issue.message).join("\n")}. Check out the configuration: https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-bluesky-posts/README.md#configuration.` ); return; } const { uris, fetchThread, threadDepth, threadParentHeight, fetchOnlyAuthorReplies, ...renderPostAsHtmlConfig } = parsedConfig.data; if (uris.length === 0) { logger.warn("No AT-URIs provided and no posts will be loaded"); return; } const preUris = meta.get("uris"); if (preUris && preUris === generateDigest(JSON.stringify(uris))) { logger.info("`uris` unchanged, skipping"); return; } meta.set("uris", generateDigest(JSON.stringify(uris))); try { const agent = new AtpAgent({ service: "https://public.api.bsky.app" }); const atUris = await Promise.all( uris.map((uri) => getAtUri(uri, agent)) ); if (!fetchThread) { logger.info(`Loading ${atUris.length} posts`); const chunkSize = 25; const allPosts = []; for (let i = 0; i < atUris.length; i += chunkSize) { const chunk = atUris.slice(i, i + chunkSize); const getPostsRes = await agent.getPosts({ uris: chunk }); if (getPostsRes.success) { allPosts.push(...getPostsRes.data.posts); } else { throw new Error( `Loading posts ${i + 1} to ${i + chunk.length} encountered unknown errors` ); } } for (const item of allPosts) { const link = atUriToPostUri(item.uri); const html = renderPostAsHtml( item.record, renderPostAsHtmlConfig ); const parsedItem = await parseData({ id: item.uri, // convert `item` to a pure POJO by stripping non-serializable, non-enumerable, and inherited properties, // preventing serialization errors 'Cannot stringify arbitrary non-POJOs' caused by devalue library. data: JSON.parse(JSON.stringify({ ...item, link, html })) }); store.set({ id: item.uri, data: parsedItem, digest: generateDigest(parsedItem), rendered: { html } }); } logger.info("Successfully loaded all posts"); } else { let count = atUris.length; logger.info( `Loading ${count} posts and ${fetchOnlyAuthorReplies ? "direct replies" : "threads"}` ); for (const uri of atUris) { const getPostThreadRes = await agent.getPostThread({ uri, depth: threadDepth, parentHeight: fetchOnlyAuthorReplies ? 0 : threadParentHeight }); if (getPostThreadRes.success) { const { data: { thread } } = getPostThreadRes; if (isNotFoundPost(thread)) { logger.warn(`Post with '${uri}' not found`); count--; continue; } if (isBlockedPost(thread)) { logger.warn(`Post with '${uri}' is blocked`); count--; continue; } if (isThreadViewPost(thread)) { const post = thread.post; const replies = thread.replies; const link = atUriToPostUri(post.uri); const html = renderPostAsHtml( post.record, renderPostAsHtmlConfig ); const parsedDate = await parseData({ id: post.uri, data: JSON.parse( JSON.stringify({ uri: post.uri, post: { ...post, link, html }, replies: fetchOnlyAuthorReplies ? getOnlyAuthorReplies( replies, threadDepth, post.author.did ).map((item) => ({ ...item, link: atUriToPostUri(item.uri), html: renderPostAsHtml( item.record, renderPostAsHtmlConfig ) })) : replies, ...fetchOnlyAuthorReplies ? {} : { parent: thread.parent } }) ) }); store.set({ id: parsedDate.uri, data: parsedDate, digest: generateDigest(parsedDate), rendered: { html: renderPostAsHtml( parsedDate.post, renderPostAsHtmlConfig ) } }); } } else { logger.warn(`Post with '${uri}' load failed`); count--; } } logger.info( `Successfully loaded ${count === atUris.length ? "all" : `${count}`} posts` ); } } catch (error) { logger.error(`Failed to load posts. ${error.message}`); } } }; } export { atUriToPostUri, blueskyPostsLoader, renderPostAsHtml };