@rsweeten/dropbox-sync
Version:
Reusable Dropbox synchronization module with framework adapters
360 lines (359 loc) • 15.5 kB
JavaScript
import path from 'path';
import fs from 'fs';
export function createSyncMethods(getClient, credentials, socket) {
let syncCancelled = false;
/**
* Normalize path for consistent comparison between local and Dropbox paths
*/
function normalizePath(filePath) {
// Remove any leading slashes for consistency, then add a single leading slash
return '/' + filePath.replace(/^\/+/, '').toLowerCase();
}
/**
* Convert blob/buffer types to Buffer
*/
async function blobToBuffer(data) {
// If it's already a Buffer, return it directly
if (Buffer.isBuffer(data)) {
return data;
}
// If it's a Blob, convert it to a Buffer
if (typeof Blob !== 'undefined' && data instanceof Blob) {
// In browser or when Blob has arrayBuffer method
if (data.arrayBuffer && typeof data.arrayBuffer === 'function') {
const arrayBuffer = await data.arrayBuffer();
return Buffer.from(arrayBuffer);
}
// Fallback for environments where arrayBuffer might not exist
return Buffer.from(String(data));
}
// If it's a plain ArrayBuffer
if (data instanceof ArrayBuffer) {
return Buffer.from(data);
}
// If it's a string
if (typeof data === 'string') {
return Buffer.from(data);
}
// For Jest mock objects that mimic Blob behavior
if (data &&
typeof data === 'object' &&
'arrayBuffer' in data &&
typeof data.arrayBuffer === 'function') {
const arrayBuffer = await data.arrayBuffer();
return Buffer.from(arrayBuffer);
}
// If we don't know what it is, throw an error
throw new Error(`Unsupported data type: ${typeof data}`);
}
/**
* Recursively scan local directory for files to sync
*/
async function scanLocalDirectory(dir, baseDir = '') {
const files = [];
try {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
const relativePath = path.join(baseDir, entry.name);
if (entry.isDirectory()) {
files.push(...(await scanLocalDirectory(fullPath, relativePath)));
}
else if (/\.(jpg|jpeg|png|gif|bmp|webp|svg|json)$/i.test(entry.name)) {
// Only include image files by default
files.push('/' + relativePath.replace(/\\/g, '/'));
}
}
}
catch (error) {
console.error(`Error scanning directory ${dir}:`, error);
}
return files;
}
/**
* Recursively scan Dropbox folder for files
*/
async function scanDropboxFolder(dropboxPath = '') {
const dropbox = getClient();
const files = [];
async function fetchFolderContents(path) {
try {
const response = (await dropbox.filesListFolder({
path,
recursive: true,
}));
for (const entry of response.result.entries) {
if (entry['.tag'] === 'file' &&
entry.path_display &&
/\.(jpg|jpeg|png|gif|bmp|webp|svg|json)$/i.test(entry.path_display)) {
files.push(entry.path_display);
}
}
if (response.result.has_more) {
await continueListing(response.result.cursor);
}
}
catch (error) {
console.error('Error fetching Dropbox folder contents:', error);
throw error;
}
}
async function continueListing(cursor) {
const response = (await dropbox.filesListFolderContinue({
cursor: cursor,
}));
for (const entry of response.result.entries) {
if (entry['.tag'] === 'file' &&
entry.path_display &&
/\.(jpg|jpeg|png|gif|bmp|webp|svg|json)$/i.test(entry.path_display)) {
files.push(entry.path_display);
}
}
if (response.result.has_more) {
await continueListing(response.result.cursor);
}
}
await fetchFolderContents(dropboxPath);
return files;
}
return {
/**
* Scan local directory for files
*/
async scanLocalFiles(dir) {
if (!dir && typeof window === 'undefined') {
throw new Error('Local directory must be provided in Node.js environment');
}
const localDir = dir || path.join(process.cwd(), 'public', 'img');
return scanLocalDirectory(localDir);
},
/**
* Scan Dropbox folder for files
*/
async scanDropboxFiles(dir) {
const dropboxDir = dir || '';
return scanDropboxFolder(dropboxDir);
},
/**
* Compare local and Dropbox files and create a sync queue
*/
async createSyncQueue(options) {
const localDir = options?.localDir || path.join(process.cwd(), 'public', 'img');
const dropboxDir = options?.dropboxDir || '';
// Use the public API methods instead of internal functions
const localFiles = await this.scanLocalFiles(localDir);
const dropboxFiles = await this.scanDropboxFiles(dropboxDir);
// Normalize paths for comparison
const normalizedDropboxFiles = dropboxFiles.map(normalizePath);
const normalizedLocalFiles = localFiles.map(normalizePath);
// Files to upload (in local but not in Dropbox)
const uploadQueue = localFiles.filter((localFile) => {
const normalizedLocalFile = normalizePath(localFile);
return !normalizedDropboxFiles.includes(normalizedLocalFile);
});
// Files to download (in Dropbox but not in local)
const downloadQueue = dropboxFiles.filter((dropboxFile) => {
const normalizedDropboxFile = normalizePath(dropboxFile);
return !normalizedLocalFiles.includes(normalizedDropboxFile);
});
return { uploadQueue, downloadQueue };
},
/**
* Synchronize files between local filesystem and Dropbox
*/
async syncFiles(options) {
const localDir = options?.localDir || path.join(process.cwd(), 'public', 'img');
const dropboxDir = options?.dropboxDir || '';
const progressCallback = options?.progressCallback;
syncCancelled = false;
const result = {
uploaded: [],
downloaded: [],
errors: [],
};
// Report initial progress
const reportProgress = (progress) => {
if (progressCallback) {
progressCallback(progress);
}
// Also emit via Socket.IO if available
if (progress.progress !== undefined && progress.message) {
socket.emit('sync:progress', {
type: progress.type || 'progress',
message: progress.message,
progress: progress.progress,
socketId: 'dropbox-sync-module',
});
}
};
reportProgress({
stage: 'init',
progress: 0,
message: 'Starting Dropbox synchronization',
});
try {
// Get the Dropbox client
const dropbox = getClient();
// Get files to sync
reportProgress({
stage: 'listing',
message: 'Analyzing files to sync...',
progress: 0,
});
const { uploadQueue, downloadQueue } = await this.createSyncQueue({ localDir, dropboxDir });
const totalTasks = uploadQueue.length + downloadQueue.length;
// Socket notification about queue
socket.emit('sync:queue', {
total: totalTasks,
totalUploads: uploadQueue.length,
totalDownloads: downloadQueue.length,
});
if (totalTasks === 0) {
reportProgress({
stage: 'complete',
progress: 100,
message: 'All files are already in sync',
});
socket.emit('sync:complete', {
message: 'All files are already in sync',
});
return result;
}
let completedTasks = 0;
// Upload files to Dropbox
for (const file of uploadQueue) {
if (syncCancelled) {
break;
}
try {
reportProgress({
stage: 'processing',
type: 'upload',
current: completedTasks + 1,
total: totalTasks,
progress: Math.round((completedTasks / totalTasks) * 100),
message: `Uploading: ${file}`,
item: file,
});
// Upload logic
const localFilePath = path.join(localDir, file.replace(/^\/+/, ''));
const dropboxFilePath = '/' + file.replace(/^\/+/, '');
const fileContent = fs.readFileSync(localFilePath);
await dropbox.filesUpload({
path: dropboxFilePath,
contents: fileContent,
mode: { '.tag': 'overwrite' },
});
result.uploaded.push(file);
completedTasks++;
}
catch (error) {
result.errors.push({ file, error: error });
console.error(`Error uploading file ${file}:`, error);
}
}
// Download files from Dropbox
for (const file of downloadQueue) {
if (syncCancelled) {
break;
}
try {
reportProgress({
stage: 'processing',
type: 'download',
current: completedTasks + 1,
total: totalTasks,
progress: Math.round((completedTasks / totalTasks) * 100),
message: `Downloading: ${file}`,
item: file,
});
// Download logic
const dropboxFilePath = file;
const localRelativePath = file.replace(/^\/+/, '');
const localFilePath = path.join(localDir, localRelativePath);
// Download the file from Dropbox with proper error handling
const response = (await dropbox.filesDownload({
path: dropboxFilePath,
}));
// Get file content - handle different possible response formats
let fileContent;
const responseResult = response.result;
if (responseResult.fileBlob) {
fileContent = await blobToBuffer(responseResult.fileBlob);
}
else if (responseResult.fileBinary) {
fileContent = Buffer.from(responseResult.fileBinary, 'binary');
}
else {
// For Node.js environment, Dropbox might return content in different properties
const contentKey = Object.keys(responseResult).find((key) => [
'content',
'fileContent',
'file',
'data',
].includes(key));
if (contentKey) {
fileContent = Buffer.from(responseResult[contentKey]);
}
else {
throw new Error(`Could not find file content in Dropbox response for ${dropboxFilePath}`);
}
}
// Create directory if it doesn't exist
const dirPath = path.dirname(localFilePath);
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
}
// Write the file to disk
fs.writeFileSync(localFilePath, fileContent);
result.downloaded.push(file);
completedTasks++;
}
catch (error) {
result.errors.push({ file, error: error });
console.error(`Error downloading file ${file}:`, error);
// Emit error but continue with the next file
socket.emit('sync:error', {
message: `Error downloading ${file}: ${error.message}`,
continue: true,
});
}
}
// Report completion
reportProgress({
stage: 'complete',
progress: 100,
message: 'Synchronization completed successfully',
});
// Sync complete notification
socket.emit('sync:complete', {
message: 'Synchronization completed successfully',
stats: {
uploaded: result.uploaded.length,
downloaded: result.downloaded.length,
},
});
}
catch (error) {
reportProgress({
stage: 'error',
message: `Error: ${error.message}`,
});
// Emit error notification
socket.emit('sync:error', {
message: error.message,
});
// Re-throw the error for handling by the caller
throw error;
}
return result;
},
/**
* Cancel an ongoing synchronization
*/
cancelSync() {
syncCancelled = true;
socket.emit('sync:cancel', {});
},
};
}