@stackbit/cms-contentful/src/content-poller.ts

Stackbit Contentful CMS Interface

import _ from 'lodash';
import { createClient, ContentfulClientApi, Entry, Asset } from 'contentful';
import { AssetProps, EntryProps, ContentTypeProps, KeyValueMap, PlainClientAPI } from 'contentful-management';
import { Logger } from '@stackbit/types';
import { createPlainApiClient, fetchContentTypesUpdatedAfter, fetchAssetsUpdatedAfter, fetchEntriesUpdatedAfter } from './contentful-api-client';
import { getLastUpdatedEntityDate } from './utils';

export type ContentPollerSyncContext = {
    lastUpdatedEntryDate?: string;
    lastUpdatedAssetDate?: string;
    lastUpdatedContentTypeDate?: string;
};

export interface ContentPollerSyncResult {
    entries: EntryProps<any>[];
    assets: AssetProps[];
    contentTypes: ContentTypeProps[];
    deletedEntries: Entry<any>[];
    deletedAssets: Asset[];
}

export type SyncCallback = (result: ContentPollerSyncResult) => Promise<void>;

export class ContentPoller {
    private readonly logger: Logger;
    private readonly client: ContentfulClientApi;
    private readonly managementClient: PlainClientAPI;
    private readonly notificationCallback: SyncCallback;
    private readonly pollingIntervalMs: number;
    private nextSyncToken: null | string;
    private running: boolean;
    private pollTimeout: NodeJS.Timeout | null;
    public readonly pollType: 'sync' | 'date';
    private syncContext!: Required<ContentPollerSyncContext>;

    constructor({
        spaceId,
        environment,
        previewToken,
        managementToken,
        previewHost,
        managementHost,
        uploadHost,
        pollingIntervalMs = 1000,
        pollType,
        syncContext,
        notificationCallback,
        logger
    }: {
        spaceId: string;
        environment: string;
        previewToken: string;
        managementToken: string;
        previewHost?: string;
        managementHost?: string;
        uploadHost?: string;
        pollingIntervalMs?: number;
        pollType?: 'sync' | 'date';
        syncContext?: ContentPollerSyncContext;
        notificationCallback: SyncCallback;
        logger: Logger;
    }) {
        this.logger = logger;
        this.nextSyncToken = null;
        this.running = false;
        this.pollTimeout = null;
        this.pollingIntervalMs = pollingIntervalMs;
        this.pollType = pollType ?? 'date';
        this.notificationCallback = notificationCallback;
        this.setSyncContext(syncContext);
        this.client = createClient({
            space: spaceId,
            environment: environment,
            accessToken: previewToken,
            host: previewHost ?? 'preview.contentful.com'
        });
        this.managementClient = createPlainApiClient({
            accessToken: managementToken,
            spaceId,
            environment,
            managementHost,
            uploadHost
        });
        this.handleTimeout = this.handleTimeout.bind(this);
    }

    start() {
        this.logger.debug(`start polling contentful for content changes using ${this.pollType}`);
        if (!this.running) {
            this.running = true;
            this.setPollTimeout();
        }
    }

    stop() {
        this.logger.debug(`stop polling contentful for content changes`);
        if (this.pollTimeout) {
            clearTimeout(this.pollTimeout);
            this.pollTimeout = null;
        }
        this.running = false;
    }

    setSyncContext(syncContext?: ContentPollerSyncContext) {
        const now = new Date().toISOString();
        this.syncContext = {
            lastUpdatedEntryDate: syncContext?.lastUpdatedEntryDate ?? now,
            lastUpdatedAssetDate: syncContext?.lastUpdatedAssetDate ?? now,
            lastUpdatedContentTypeDate: syncContext?.lastUpdatedContentTypeDate ?? now
        };
    }

    private setPollTimeout() {
        if (this.pollTimeout) {
            clearTimeout(this.pollTimeout);
        }
        this.pollTimeout = setTimeout(this.handleTimeout, this.pollingIntervalMs);
    }

    async handleTimeout() {
        this.pollTimeout = null;
        try {
            if (this.pollType === 'sync') {
                await this.pollSync();
            } else {
                await this.pollDate();
            }
        } catch (error: any) {
            this.logger.warn('error polling', { error: error.message });
        }
        if (this.running) {
            this.setPollTimeout();
        }
    }

    async pollDate(): Promise<any> {
        const prevSyncContext = this.syncContext;
        const entries = await fetchEntriesUpdatedAfter(this.managementClient, this.syncContext.lastUpdatedEntryDate);
        const assets = await fetchAssetsUpdatedAfter(this.managementClient, this.syncContext.lastUpdatedAssetDate);
        const contentTypes = await fetchContentTypesUpdatedAfter(this.managementClient, this.syncContext.lastUpdatedContentTypeDate);
        // Check that syncContext wasn't updated externally via setSyncContext() while the content was fetched.
        // If it was updated (prevSyncContext != this.syncContext), then ignore the dates from the fetched content
        // and keep the syncContext that was set externally via setSyncContext().
        if (_.isEqual(prevSyncContext, this.syncContext)) {
            this.syncContext = _.defaults(
                {
                    lastUpdatedEntryDate: getLastUpdatedEntityDate(entries),
                    lastUpdatedAssetDate: getLastUpdatedEntityDate(assets),
                    lastUpdatedContentTypeDate: getLastUpdatedEntityDate(contentTypes)
                },
                this.syncContext
            );
        }
        if (contentTypes.length || entries.length || assets.length) {
            this.logger.info(
                `poll date response: got ` +
                    `${entries.length} changed entries, ` +
                    `${assets.length} changed assets, ` +
                    `${contentTypes.length} changed content types`
            );
            await this.notificationCallback({
                contentTypes,
                entries,
                assets,
                deletedEntries: [],
                deletedAssets: []
            });
        }
    }

    async pollSync(): Promise<any> {
        const initial = this.nextSyncToken === null;
        let hasMoreItems = true;
        let hasItems = false;
        const result: ContentPollerSyncResult = {
            entries: [],
            assets: [],
            deletedEntries: [],
            deletedAssets: [],
            contentTypes: []
        };
        if (initial) {
            this.logger.info('Running initial sync...');
        }
        while (hasMoreItems) {
            const response = await this.client.sync({
                initial: this.nextSyncToken === null,
                nextSyncToken: this.nextSyncToken,
                resolveLinks: false,
                limit: 1000
            });
            this.logger.info(`sync response: ${response.entries.length}`);
            const isEmptyResponse = _.every(_.pick(response, ['entries', 'assets', 'deletedEntries', 'deletedAssets']), _.isEmpty);
            if (this.nextSyncToken === response.nextSyncToken || isEmptyResponse) {
                hasMoreItems = false;
            } else {
                if (!initial) {
                    // refetch entries from management api to get full sys data
                    const { entries, assets } = await this.batchRefetchData({
                        entryIds: response.entries.map((entry) => entry.sys.id),
                        assetIds: response.assets.map((entry) => entry.sys.id)
                    });
                    hasItems = true;
                    result.entries = result.entries.concat(entries);
                    result.assets = result.assets.concat(assets);
                    result.deletedEntries = result.deletedEntries.concat(response.deletedEntries);
                    result.deletedAssets = result.deletedAssets.concat(response.deletedAssets);
                }
            }
            this.nextSyncToken = response.nextSyncToken;
        }
        if (!initial && hasItems) {
            await this.notificationCallback(result);
        }
        if (initial) {
            this.logger.info('Initial sync done.');
        }
    }

    private async batchRefetchData({
        entryIds,
        assetIds
    }: {
        entryIds: string[];
        assetIds: string[];
    }): Promise<{ entries: EntryProps<KeyValueMap>[]; assets: AssetProps[] }> {
        const limit = 300;
        const entryChunks = _.chunk(entryIds, limit);
        const entries = _.flatMap(
            await Promise.all(
                entryChunks.map((chunk) =>
                    this.managementClient.entry.getMany({
                        query: {
                            limit: limit,
                            'sys.id[in]': chunk.join(',')
                        }
                    })
                )
            ),
            (result) => result.items
        );

        const assetChunks = _.chunk(assetIds, limit);
        const assets = _.flatMap(
            await Promise.all(
                assetChunks.map((chunk) =>
                    this.managementClient.asset.getMany({
                        query: {
                            limit: limit,
                            'sys.id[in]': chunk.join(',')
                        }
                    })
                )
            ),
            (result) => result.items
        );

        return { entries, assets };
    }
}