@rsweeten/dropbox-sync/src/core/sync.ts

Reusable Dropbox synchronization module with framework adapters

import path from 'path'
import fs from 'fs'
import { Dropbox } from 'dropbox'
import type {
    DropboxCredentials,
    SyncMethods,
    SyncOptions,
    SyncResult,
    SyncProgress,
    ProgressCallback,
    SocketMethods,
} from './types'

// Define interface for Dropbox file entries
interface DropboxFileEntry {
    '.tag': string
    path_display?: string
    [key: string]: any
}

// Define interface for Dropbox API responses
interface DropboxListFolderResponse {
    result: {
        entries: DropboxFileEntry[]
        has_more: boolean
        cursor: string
    }
}

interface DropboxDownloadResponse {
    result: {
        fileBlob?: Blob
        fileBinary?: string
        [key: string]: any
    }
}

export function createSyncMethods(
    getClient: () => Dropbox,
    credentials: DropboxCredentials,
    socket: SocketMethods
): SyncMethods {
    let syncCancelled = false

    /**
     * Normalize path for consistent comparison between local and Dropbox paths
     */
    function normalizePath(filePath: string): string {
        // Remove any leading slashes for consistency, then add a single leading slash
        return '/' + filePath.replace(/^\/+/, '').toLowerCase()
    }

    /**
     * Convert blob/buffer types to Buffer
     */
    async function blobToBuffer(data: any): Promise<Buffer> {
        // If it's already a Buffer, return it directly
        if (Buffer.isBuffer(data)) {
            return data
        }

        // If it's a Blob, convert it to a Buffer
        if (typeof Blob !== 'undefined' && data instanceof Blob) {
            // In browser or when Blob has arrayBuffer method
            if (data.arrayBuffer && typeof data.arrayBuffer === 'function') {
                const arrayBuffer = await data.arrayBuffer()
                return Buffer.from(arrayBuffer)
            }
            // Fallback for environments where arrayBuffer might not exist
            return Buffer.from(String(data))
        }

        // If it's a plain ArrayBuffer
        if (data instanceof ArrayBuffer) {
            return Buffer.from(data)
        }

        // If it's a string
        if (typeof data === 'string') {
            return Buffer.from(data)
        }

        // For Jest mock objects that mimic Blob behavior
        if (
            data &&
            typeof data === 'object' &&
            'arrayBuffer' in data &&
            typeof data.arrayBuffer === 'function'
        ) {
            const arrayBuffer = await data.arrayBuffer()
            return Buffer.from(arrayBuffer)
        }

        // If we don't know what it is, throw an error
        throw new Error(`Unsupported data type: ${typeof data}`)
    }

    /**
     * Recursively scan local directory for files to sync
     */
    async function scanLocalDirectory(
        dir: string,
        baseDir = ''
    ): Promise<string[]> {
        const files: string[] = []

        try {
            const entries = fs.readdirSync(dir, { withFileTypes: true })

            for (const entry of entries) {
                const fullPath = path.join(dir, entry.name)
                const relativePath = path.join(baseDir, entry.name)

                if (entry.isDirectory()) {
                    files.push(
                        ...(await scanLocalDirectory(fullPath, relativePath))
                    )
                } else if (
                    /\.(jpg|jpeg|png|gif|bmp|webp|svg|json)$/i.test(entry.name)
                ) {
                    // Only include image files by default
                    files.push('/' + relativePath.replace(/\\/g, '/'))
                }
            }
        } catch (error) {
            console.error(`Error scanning directory ${dir}:`, error)
        }

        return files
    }

    /**
     * Recursively scan Dropbox folder for files
     */
    async function scanDropboxFolder(
        dropboxPath: string = ''
    ): Promise<string[]> {
        const dropbox = getClient()
        const files: string[] = []

        async function fetchFolderContents(path: string) {
            try {
                const response = (await dropbox.filesListFolder({
                    path,
                    recursive: true,
                })) as unknown as DropboxListFolderResponse

                for (const entry of response.result.entries) {
                    if (
                        entry['.tag'] === 'file' &&
                        entry.path_display &&
                        /\.(jpg|jpeg|png|gif|bmp|webp|svg|json)$/i.test(
                            entry.path_display
                        )
                    ) {
                        files.push(entry.path_display)
                    }
                }

                if (response.result.has_more) {
                    await continueListing(response.result.cursor)
                }
            } catch (error) {
                console.error('Error fetching Dropbox folder contents:', error)
                throw error
            }
        }

        async function continueListing(cursor: string) {
            const response = (await dropbox.filesListFolderContinue({
                cursor: cursor,
            })) as unknown as DropboxListFolderResponse

            for (const entry of response.result.entries) {
                if (
                    entry['.tag'] === 'file' &&
                    entry.path_display &&
                    /\.(jpg|jpeg|png|gif|bmp|webp|svg|json)$/i.test(
                        entry.path_display
                    )
                ) {
                    files.push(entry.path_display)
                }
            }

            if (response.result.has_more) {
                await continueListing(response.result.cursor)
            }
        }

        await fetchFolderContents(dropboxPath)
        return files
    }

    return {
        /**
         * Scan local directory for files
         */
        async scanLocalFiles(dir?: string): Promise<string[]> {
            if (!dir && typeof window === 'undefined') {
                throw new Error(
                    'Local directory must be provided in Node.js environment'
                )
            }

            const localDir = dir || path.join(process.cwd(), 'public', 'img')
            return scanLocalDirectory(localDir)
        },

        /**
         * Scan Dropbox folder for files
         */
        async scanDropboxFiles(dir?: string): Promise<string[]> {
            const dropboxDir = dir || ''
            return scanDropboxFolder(dropboxDir)
        },

        /**
         * Compare local and Dropbox files and create a sync queue
         */
        async createSyncQueue(
            options?: Partial<SyncOptions>
        ): Promise<{ uploadQueue: string[]; downloadQueue: string[] }> {
            const localDir =
                options?.localDir || path.join(process.cwd(), 'public', 'img')
            const dropboxDir = options?.dropboxDir || ''

            // Use the public API methods instead of internal functions
            const localFiles = await this.scanLocalFiles(localDir)
            const dropboxFiles = await this.scanDropboxFiles(dropboxDir)

            // Normalize paths for comparison
            const normalizedDropboxFiles = dropboxFiles.map(normalizePath)
            const normalizedLocalFiles = localFiles.map(normalizePath)

            // Files to upload (in local but not in Dropbox)
            const uploadQueue = localFiles.filter((localFile) => {
                const normalizedLocalFile = normalizePath(localFile)
                return !normalizedDropboxFiles.includes(normalizedLocalFile)
            })

            // Files to download (in Dropbox but not in local)
            const downloadQueue = dropboxFiles.filter((dropboxFile) => {
                const normalizedDropboxFile = normalizePath(dropboxFile)
                return !normalizedLocalFiles.includes(normalizedDropboxFile)
            })

            return { uploadQueue, downloadQueue }
        },

        /**
         * Synchronize files between local filesystem and Dropbox
         */
        async syncFiles(options?: Partial<SyncOptions>): Promise<SyncResult> {
            const localDir =
                options?.localDir || path.join(process.cwd(), 'public', 'img')
            const dropboxDir = options?.dropboxDir || ''
            const progressCallback = options?.progressCallback

            syncCancelled = false
            const result: SyncResult = {
                uploaded: [],
                downloaded: [],
                errors: [],
            }

            // Report initial progress
            const reportProgress = (progress: SyncProgress) => {
                if (progressCallback) {
                    progressCallback(progress)
                }

                // Also emit via Socket.IO if available
                if (progress.progress !== undefined && progress.message) {
                    socket.emit('sync:progress', {
                        type: progress.type || 'progress',
                        message: progress.message,
                        progress: progress.progress,
                        socketId: 'dropbox-sync-module',
                    })
                }
            }

            reportProgress({
                stage: 'init',
                progress: 0,
                message: 'Starting Dropbox synchronization',
            })

            try {
                // Get the Dropbox client
                const dropbox = getClient()

                // Get files to sync
                reportProgress({
                    stage: 'listing',
                    message: 'Analyzing files to sync...',
                    progress: 0,
                })

                const { uploadQueue, downloadQueue } =
                    await this.createSyncQueue({ localDir, dropboxDir })
                const totalTasks = uploadQueue.length + downloadQueue.length

                // Socket notification about queue
                socket.emit('sync:queue', {
                    total: totalTasks,
                    totalUploads: uploadQueue.length,
                    totalDownloads: downloadQueue.length,
                })

                if (totalTasks === 0) {
                    reportProgress({
                        stage: 'complete',
                        progress: 100,
                        message: 'All files are already in sync',
                    })

                    socket.emit('sync:complete', {
                        message: 'All files are already in sync',
                    })

                    return result
                }

                let completedTasks = 0

                // Upload files to Dropbox
                for (const file of uploadQueue) {
                    if (syncCancelled) {
                        break
                    }

                    try {
                        reportProgress({
                            stage: 'processing',
                            type: 'upload',
                            current: completedTasks + 1,
                            total: totalTasks,
                            progress: Math.round(
                                (completedTasks / totalTasks) * 100
                            ),
                            message: `Uploading: ${file}`,
                            item: file,
                        })

                        // Upload logic
                        const localFilePath = path.join(
                            localDir,
                            file.replace(/^\/+/, '')
                        )
                        const dropboxFilePath = '/' + file.replace(/^\/+/, '')
                        const fileContent = fs.readFileSync(localFilePath)

                        await dropbox.filesUpload({
                            path: dropboxFilePath,
                            contents: fileContent,
                            mode: { '.tag': 'overwrite' },
                        })

                        result.uploaded.push(file)
                        completedTasks++
                    } catch (error) {
                        result.errors.push({ file, error: error as Error })
                        console.error(`Error uploading file ${file}:`, error)
                    }
                }

                // Download files from Dropbox
                for (const file of downloadQueue) {
                    if (syncCancelled) {
                        break
                    }

                    try {
                        reportProgress({
                            stage: 'processing',
                            type: 'download',
                            current: completedTasks + 1,
                            total: totalTasks,
                            progress: Math.round(
                                (completedTasks / totalTasks) * 100
                            ),
                            message: `Downloading: ${file}`,
                            item: file,
                        })

                        // Download logic
                        const dropboxFilePath = file
                        const localRelativePath = file.replace(/^\/+/, '')
                        const localFilePath = path.join(
                            localDir,
                            localRelativePath
                        )

                        // Download the file from Dropbox with proper error handling
                        const response = (await dropbox.filesDownload({
                            path: dropboxFilePath,
                        })) as unknown as DropboxDownloadResponse

                        // Get file content - handle different possible response formats
                        let fileContent: Buffer
                        const responseResult = response.result

                        if (responseResult.fileBlob) {
                            fileContent = await blobToBuffer(
                                responseResult.fileBlob
                            )
                        } else if (responseResult.fileBinary) {
                            fileContent = Buffer.from(
                                responseResult.fileBinary,
                                'binary'
                            )
                        } else {
                            // For Node.js environment, Dropbox might return content in different properties
                            const contentKey = Object.keys(responseResult).find(
                                (key) =>
                                    [
                                        'content',
                                        'fileContent',
                                        'file',
                                        'data',
                                    ].includes(key)
                            )

                            if (contentKey) {
                                fileContent = Buffer.from(
                                    responseResult[contentKey]
                                )
                            } else {
                                throw new Error(
                                    `Could not find file content in Dropbox response for ${dropboxFilePath}`
                                )
                            }
                        }

                        // Create directory if it doesn't exist
                        const dirPath = path.dirname(localFilePath)
                        if (!fs.existsSync(dirPath)) {
                            fs.mkdirSync(dirPath, { recursive: true })
                        }

                        // Write the file to disk
                        fs.writeFileSync(localFilePath, fileContent)

                        result.downloaded.push(file)
                        completedTasks++
                    } catch (error) {
                        result.errors.push({ file, error: error as Error })
                        console.error(`Error downloading file ${file}:`, error)

                        // Emit error but continue with the next file
                        socket.emit('sync:error', {
                            message: `Error downloading ${file}: ${
                                (error as Error).message
                            }`,
                            continue: true,
                        })
                    }
                }

                // Report completion
                reportProgress({
                    stage: 'complete',
                    progress: 100,
                    message: 'Synchronization completed successfully',
                })

                // Sync complete notification
                socket.emit('sync:complete', {
                    message: 'Synchronization completed successfully',
                    stats: {
                        uploaded: result.uploaded.length,
                        downloaded: result.downloaded.length,
                    },
                })
            } catch (error) {
                reportProgress({
                    stage: 'error',
                    message: `Error: ${(error as Error).message}`,
                })

                // Emit error notification
                socket.emit('sync:error', {
                    message: (error as Error).message,
                })

                // Re-throw the error for handling by the caller
                throw error
            }

            return result
        },

        /**
         * Cancel an ongoing synchronization
         */
        cancelSync() {
            syncCancelled = true
            socket.emit('sync:cancel', {})
        },
    }
}