@syngrisi/syngrisi
Version:
Syngrisi - Visual Testing Tool
286 lines • 10.2 kB
JavaScript
// src/tasks/lib/dataBackupRestore.ts
import fs from "fs";
import { promises as fsp } from "fs";
import path from "path";
import { createGzip, createGunzip } from "zlib";
import { promisify } from "util";
import { pipeline } from "stream";
import tar from "tar-stream";
import mongoose from "mongoose";
var pipelineAsync = promisify(pipeline);
var { BSON } = mongoose.mongo;
async function ensureDir(dirPath) {
await fsp.mkdir(dirPath, { recursive: true });
}
async function addFileToTar(pack, filePath, entryName) {
const stat = await fsp.stat(filePath);
await new Promise((resolve, reject) => {
const entry = pack.entry({ name: entryName, size: stat.size, mode: stat.mode }, (error) => {
if (error) {
reject(error);
return;
}
resolve();
});
fs.createReadStream(filePath).on("error", reject).pipe(entry).on("error", reject);
});
}
async function walkFiles(rootDir, onFile) {
const stack = [rootDir];
while (stack.length > 0) {
const currentDir = stack.pop();
if (!currentDir) continue;
const dir = await fsp.opendir(currentDir);
for await (const entry of dir) {
const fullPath = path.join(currentDir, entry.name);
const relativePath = path.relative(rootDir, fullPath);
if (entry.isDirectory()) {
stack.push(fullPath);
} else if (entry.isFile()) {
await onFile(fullPath, relativePath);
}
}
}
}
async function createTarGzArchive(outputPath, items) {
await ensureDir(path.dirname(outputPath));
const pack = tar.pack();
const gzip = createGzip();
const output = fs.createWriteStream(outputPath);
const archivePipeline = pipelineAsync(pack, gzip, output);
for (const item of items) {
await addFileToTar(pack, item.path, item.name);
}
pack.finalize();
await archivePipeline;
}
async function extractTarGzArchive(archivePath, destinationDir) {
await ensureDir(destinationDir);
const extract = tar.extract();
await new Promise((resolve, reject) => {
extract.on("entry", (header, stream, next) => {
const outputPath = path.join(destinationDir, header.name);
const finish = (error) => {
if (error) {
reject(error);
return;
}
next();
};
if (header.type === "directory") {
void ensureDir(outputPath).then(() => {
stream.resume();
finish();
}).catch((error) => finish(error));
return;
}
void ensureDir(path.dirname(outputPath)).then(() => pipelineAsync(stream, fs.createWriteStream(outputPath))).then(() => finish()).catch((error) => finish(error));
});
extract.on("finish", () => resolve());
extract.on("error", reject);
fs.createReadStream(archivePath).on("error", reject).pipe(createGunzip()).on("error", reject).pipe(extract).on("error", reject);
});
}
async function writeCollectionDump(connection, collectionName, outputPath) {
const db = connection.db;
if (!db) {
throw new Error("MongoDB connection is not available");
}
await ensureDir(path.dirname(outputPath));
const gzip = createGzip();
const output = fs.createWriteStream(outputPath);
gzip.pipe(output);
let documentCount = 0;
const cursor = db.collection(collectionName).find({}, { timeout: false });
for await (const doc of cursor) {
gzip.write(BSON.serialize(doc));
documentCount += 1;
}
gzip.end();
await new Promise((resolve, reject) => {
output.on("finish", () => resolve());
output.on("error", reject);
gzip.on("error", reject);
});
return documentCount;
}
async function importCollectionDump(connection, collectionName, dumpPath) {
const db = connection.db;
if (!db) {
throw new Error("MongoDB connection is not available");
}
const collection = db.collection(collectionName);
const batch = [];
const flush = async () => {
if (batch.length === 0) return;
await collection.insertMany(batch, { ordered: false });
batch.length = 0;
};
const input = fs.createReadStream(dumpPath).pipe(createGunzip());
let pending = Buffer.alloc(0);
for await (const chunk of input) {
pending = Buffer.concat([pending, chunk]);
while (pending.length >= 4) {
const documentSize = pending.readInt32LE(0);
if (documentSize <= 0) {
throw new Error(`Invalid BSON document size ${documentSize} in ${collectionName}`);
}
if (pending.length < documentSize) {
break;
}
const documentBuffer = pending.subarray(0, documentSize);
pending = pending.subarray(documentSize);
batch.push(BSON.deserialize(documentBuffer));
if (batch.length >= 1e3) {
await flush();
}
}
}
if (pending.length > 0) {
throw new Error(`Unexpected trailing BSON bytes in ${collectionName}`);
}
await flush();
}
async function recreateIndexes(connection, collectionName, indexes) {
const db = connection.db;
if (!db) {
throw new Error("MongoDB connection is not available");
}
const filtered = indexes.filter((index) => index.name !== "_id_");
if (filtered.length === 0) {
return;
}
const definitions = filtered.map((index) => {
const { key, ...options } = index;
return { key, ...options };
});
await db.collection(collectionName).createIndexes(definitions);
}
async function connectToMongo(connectionString) {
const connection = await mongoose.createConnection(connectionString).asPromise();
return connection;
}
async function createDatabaseBackupArchive(connectionString, outputPath) {
const connection = await connectToMongo(connectionString);
const exportDir = path.join(path.dirname(outputPath), "db-export");
try {
await ensureDir(exportDir);
const db = connection.db;
if (!db) {
throw new Error("MongoDB connection is not available");
}
const collectionInfos = await db.listCollections({}, { nameOnly: true }).toArray();
const collections = collectionInfos.map((item) => item.name).filter((name) => !name.startsWith("system."));
const manifest = {
format: "syngrisi-db-backup-v1",
exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
databaseName: db.databaseName,
collections: []
};
for (const collectionName of collections) {
const dumpFileName = `${collectionName}.bson.gz`;
const dumpPath = path.join(exportDir, dumpFileName);
const documentCount = await writeCollectionDump(connection, collectionName, dumpPath);
const indexes = await db.collection(collectionName).indexes();
manifest.collections.push({
name: collectionName,
dumpFile: dumpFileName,
documentCount,
indexes: indexes.map((index) => JSON.parse(JSON.stringify(index)))
});
}
const manifestPath = path.join(exportDir, "manifest.json");
await fsp.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
await createTarGzArchive(outputPath, [
{ path: manifestPath, name: "manifest.json" },
...manifest.collections.map((collection) => ({
path: path.join(exportDir, collection.dumpFile),
name: `collections/${collection.dumpFile}`
}))
]);
} finally {
await connection.close();
await fsp.rm(exportDir, { recursive: true, force: true });
}
}
async function restoreDatabaseBackupArchive(connectionString, archivePath) {
const connection = await connectToMongo(connectionString);
const extractDir = path.join(path.dirname(archivePath), `restore-${Date.now()}`);
try {
await extractTarGzArchive(archivePath, extractDir);
const manifestPath = path.join(extractDir, "manifest.json");
const manifest = JSON.parse(await fsp.readFile(manifestPath, "utf8"));
if (manifest.format !== "syngrisi-db-backup-v1") {
throw new Error("Unsupported database backup format");
}
const db = connection.db;
if (!db) {
throw new Error("MongoDB connection is not available");
}
await db.dropDatabase();
for (const collectionInfo of manifest.collections) {
const dumpPath = path.join(extractDir, "collections", collectionInfo.dumpFile);
await importCollectionDump(connection, collectionInfo.name, dumpPath);
await recreateIndexes(connection, collectionInfo.name, collectionInfo.indexes);
}
} finally {
await connection.close();
await fsp.rm(extractDir, { recursive: true, force: true });
}
}
async function createScreenshotsArchive(imagesPath, outputPath) {
const pack = tar.pack();
const gzip = createGzip();
const output = fs.createWriteStream(outputPath);
const archivePipeline = pipelineAsync(pack, gzip, output);
await walkFiles(imagesPath, async (fullPath, relativePath) => {
await addFileToTar(pack, fullPath, relativePath);
});
pack.finalize();
await archivePipeline;
}
async function restoreScreenshotsArchive(archivePath, destinationPath, options = {}) {
const { skipExisting = false } = options;
const extract = tar.extract();
await new Promise((resolve, reject) => {
extract.on("entry", (header, stream, next) => {
const outputPath = path.join(destinationPath, header.name);
const finish = (error) => {
if (error) {
reject(error);
return;
}
next();
};
if (header.type !== "file") {
stream.resume();
finish();
return;
}
void (async () => {
await ensureDir(path.dirname(outputPath));
if (skipExisting) {
try {
await fsp.access(outputPath);
stream.resume();
return;
} catch {
}
} else {
await fsp.rm(outputPath, { force: true });
}
await pipelineAsync(stream, fs.createWriteStream(outputPath));
})().then(() => finish()).catch((error) => finish(error));
});
extract.on("finish", () => resolve());
extract.on("error", reject);
fs.createReadStream(archivePath).on("error", reject).pipe(createGunzip()).on("error", reject).pipe(extract).on("error", reject);
});
}
export {
createDatabaseBackupArchive,
createScreenshotsArchive,
restoreDatabaseBackupArchive,
restoreScreenshotsArchive
};
//# sourceMappingURL=dataBackupRestore.js.map