kura-xmitter
Version:
The FileSystem API abstraction library, file transmitter
492 lines • 23 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Synchronizer = exports.SYNC_RESULT_FALSES = exports.Notifier = void 0;
const kura_1 = require("kura");
const DEFAULT_HANDLER = {
afterCopy: async () => {
},
afterDelete: async () => {
},
beforeCopy: () => Promise.resolve(false),
beforeDelete: () => Promise.resolve(false),
completed: async (result) => {
},
getNames: (fileNameIndex) => {
return Object.values(fileNameIndex)
.sort((a, b) => b.modified - a.modified)
.map((record) => record.name);
},
};
class Notifier {
constructor(_callback = (processed, total) => {
}) {
this._callback = _callback;
this._processed = 0;
this._total = 0;
}
get processed() {
return this._processed;
}
get total() {
return this._total;
}
incrementProcessed(count = 1) {
this._processed = this._processed + count;
this._callback(this._processed, this._total);
}
incrementTotal(count = 1) {
this._total = this._total + count;
this._callback(this._processed, this._total);
}
}
exports.Notifier = Notifier;
const DEFAULT_NOTIFIER = new Notifier();
exports.SYNC_RESULT_FALSES = {
forward: false,
backward: false,
errors: [],
};
class Synchronizer {
constructor(local, remote, options = {}) {
this.local = local;
this.remote = remote;
this.options = options;
if (options.excludeNamePattern == null) {
options.excludeNamePattern = "^\\.|^$";
}
if (options.excludePathPattern == null) {
options.excludePathPattern = "\\/\\.";
}
if (options.verbose == null)
options.verbose = false;
this.excludeNameRegExp = new RegExp(options.excludeNamePattern);
this.excludePathRegExp = new RegExp(options.excludePathPattern);
const localFS = local.filesystem;
this.localAccessor = localFS.accessor;
if (!this.localAccessor || !this.localAccessor.options.index) {
throw new Error(`Source filesystem "${localFS.name}" has no index`);
}
const remoteFS = remote.filesystem;
this.remoteAccessor = remoteFS.accessor;
if (!this.remoteAccessor || !this.remoteAccessor.options.index) {
throw new Error(`Destination filesystem "${remoteFS.name}" has no index`);
}
if (options.transferer) {
this.transferer = options.transferer;
}
else {
this.transferer = new kura_1.Transferer();
}
}
async synchronizeAll() {
return await this.synchronizeDirectory(this.local.root.fullPath, true);
}
async synchronizeDirectory(dirPath, recursively, notifier = DEFAULT_NOTIFIER, handler = DEFAULT_HANDLER) {
try {
if (!dirPath) {
dirPath = kura_1.DIR_SEPARATOR;
}
const result = await this.synchronizeChildren(this.localAccessor, this.remoteAccessor, dirPath, recursively, notifier, handler);
this.debug(this.localAccessor, this.remoteAccessor, `SyncResult: localToRemote=${result.forward}, remoteToLocal=${result.backward}`, dirPath);
await handler.completed(result);
return result;
}
catch (e) {
console.warn("synchronizeDirectory", { dirPath, recursively }, e);
const result = { ...exports.SYNC_RESULT_FALSES, errors: [e] };
await handler.completed(result, e);
return result;
}
}
async copyFile(fromAccessor, toAccessor, fromRecord, handler) {
const obj = this.toFileSystemObject(fromRecord);
if (await handler.beforeCopy(fromAccessor, toAccessor, obj)) {
return;
}
toAccessor.clearContentsCache(obj.fullPath);
await this.transferer.transfer(fromAccessor, obj, toAccessor, obj);
await toAccessor.saveRecord(obj.fullPath, {
modified: fromRecord.modified,
size: fromRecord.size,
});
await handler.afterCopy(fromAccessor, toAccessor, obj);
this.debug(fromAccessor, toAccessor, "copyFile", obj.fullPath);
}
debug(fromAccessor, toAccessor, title, path) {
if (!this.options.verbose) {
return;
}
if (fromAccessor) {
console.debug(`${fromAccessor.name} => ${toAccessor.name} - ${title}: ${path}`);
}
else {
console.debug(`${toAccessor.name} - ${title}: ${path}`);
}
}
async deleteEntry(accessor, record, handler) {
const obj = this.toFileSystemObject(record);
const fullPath = obj.fullPath;
const isFile = obj.size != null;
this.debug(null, accessor, "delete", fullPath);
try {
if (await handler.beforeDelete(accessor, obj)) {
return;
}
if (isFile) {
await accessor.delete(fullPath, true, false);
}
else {
await accessor.deleteRecursively(fullPath, false);
}
await handler.afterDelete(accessor, obj);
}
catch (e) {
if (e instanceof kura_1.NotFoundError) {
console.info(e, fullPath, "deleteEntry");
}
else {
console.warn(e, fullPath, "deleteEntry");
}
}
}
async makeDirectory(accessor, record) {
const obj = this.toFileSystemObject(record);
await accessor.doPutObject(obj);
await accessor.saveRecord(obj.fullPath, {
modified: record.modified ?? Date.now(),
});
}
mergeResult(result, merged) {
merged.forward = merged.forward || result.forward;
merged.backward = merged.backward || result.backward;
merged.errors = [...merged.errors, ...result.errors];
}
async synchronizeChildren(fromAccessor, toAccessor, dirPath, recursively, notifier, handler) {
if (this.excludePathRegExp.test(dirPath)) {
return exports.SYNC_RESULT_FALSES;
}
const fromFileNameIndex = await fromAccessor.getFileNameIndex(dirPath);
const toFileNameIndex = await toAccessor.getFileNameIndex(dirPath);
const fromNames = handler.getNames(fromFileNameIndex);
notifier.incrementTotal(fromNames.length);
const toNames = handler.getNames(toFileNameIndex);
const fromToResult = kura_1.deepCopy(exports.SYNC_RESULT_FALSES);
const toFromResult = kura_1.deepCopy(exports.SYNC_RESULT_FALSES);
outer: for (const fromName of fromNames) {
if (this.excludeNameRegExp.test(fromName)) {
notifier.incrementProcessed();
continue;
}
for (let i = 0, end = toNames.length; i < end; i++) {
const toName = toNames[i];
if (fromName !== toName) {
continue;
}
const oneResult = await this.synchronizeOne(fromAccessor, fromFileNameIndex, toAccessor, toFileNameIndex, fromName, recursively, notifier, handler);
this.mergeResult(oneResult, fromToResult);
notifier.incrementProcessed();
toNames.splice(i, 1);
continue outer;
}
const oneResult = await this.synchronizeOne(fromAccessor, fromFileNameIndex, toAccessor, toFileNameIndex, fromName, recursively, notifier, handler);
this.mergeResult(oneResult, fromToResult);
notifier.incrementProcessed();
}
notifier.incrementTotal(toNames.length);
for (const toName of toNames) {
if (this.excludeNameRegExp.test(toName)) {
notifier.incrementProcessed();
continue;
}
const oneResult = await this.synchronizeOne(toAccessor, toFileNameIndex, fromAccessor, fromFileNameIndex, toName, recursively, notifier, handler);
this.mergeResult(oneResult, toFromResult);
notifier.incrementProcessed();
}
const result = {
forward: fromToResult.forward || toFromResult.backward,
backward: fromToResult.backward || toFromResult.forward,
errors: [...fromToResult.errors, ...toFromResult.errors],
};
return result;
}
async synchronizeOne(fromAccessor, fromFileNameIndex, toAccessor, toFileNameIndex, name, recursively, notifier, handler) {
const result = kura_1.deepCopy(exports.SYNC_RESULT_FALSES);
try {
let fromRecord = fromFileNameIndex[name];
let toRecord = toFileNameIndex[name];
if (fromRecord == null && toRecord == null) {
this.warn(fromAccessor, toAccessor, name, "No records");
return result;
}
if (fromRecord != null && toRecord == null) {
toRecord = kura_1.deepCopy(fromRecord);
delete toRecord.deleted;
toRecord.modified = Synchronizer.NOT_EXISTS;
}
else if (fromRecord == null && toRecord != null) {
fromRecord = kura_1.deepCopy(toRecord);
delete fromRecord.deleted;
fromRecord.modified = Synchronizer.NOT_EXISTS;
}
const fullPath = fromRecord.fullPath;
const fromModified = fromRecord.modified;
const toModified = toRecord.modified;
if (fromRecord.size == null && toRecord.size != null) {
this.warn(fromAccessor, toAccessor, fullPath, "source is directory and destination is file");
return result;
}
else if (fromRecord.size != null && toRecord.size == null) {
this.warn(fromAccessor, toAccessor, fullPath, "source is file and destination is directory");
return result;
}
const fromDeleted = fromRecord.deleted;
const toDeleted = toRecord.deleted;
if (fromDeleted != null && toDeleted != null) {
this.debug(fromAccessor, toAccessor, (fromRecord.size != null ? "file" : "dir") + "[-from,-to]", fullPath);
return result;
}
if (fromRecord.size != null) {
if (fromDeleted != null && toDeleted == null) {
if (fromDeleted < toModified) {
try {
await this.copyFile(toAccessor, fromAccessor, toRecord, handler);
result.backward = true;
this.debug(fromAccessor, toAccessor, "file[-from,to (-from < to) => +from]", fullPath);
}
catch (e) {
if (e instanceof kura_1.NotFoundError) {
await toAccessor.deleteRecord(fullPath, true);
this.debug(fromAccessor, toAccessor, "file[-from,to (-to!) => -to]", fullPath);
}
else {
throw e;
}
}
}
else {
if (toModified !== Synchronizer.NOT_EXISTS) {
await this.deleteEntry(toAccessor, toRecord, handler);
result.forward = true;
this.debug(fromAccessor, toAccessor, "file[-from,to (to <= -from) => -to]", fullPath);
}
else {
if (fromAccessor === this.localAccessor) {
await this.truncateRecord(toAccessor, fullPath);
}
this.debug(fromAccessor, toAccessor, "file[-from,?to =>]", fullPath);
}
}
}
else if (fromDeleted == null && toDeleted != null) {
if (toDeleted < fromModified) {
try {
await this.copyFile(fromAccessor, toAccessor, fromRecord, handler);
result.forward = true;
this.debug(fromAccessor, toAccessor, "file[from,-to (-to < from) => +to]", fullPath);
}
catch (e) {
if (e instanceof kura_1.NotFoundError) {
await fromAccessor.deleteRecord(fullPath, true);
this.debug(fromAccessor, toAccessor, "file[from,-to (-from!) => -from]", fullPath);
}
else {
throw e;
}
}
}
else {
if (fromModified !== Synchronizer.NOT_EXISTS) {
await this.deleteEntry(fromAccessor, fromRecord, handler);
result.backward = true;
this.debug(toAccessor, fromAccessor, "file[from,-to (from < -to) => -from]", fullPath);
}
else {
if (toAccessor === this.localAccessor) {
await this.truncateRecord(toAccessor, fullPath);
}
this.debug(toAccessor, fromAccessor, "file[?from,-to =>]", fullPath);
}
}
}
else if (fromDeleted == null && toDeleted == null) {
if (toModified < fromModified) {
try {
await this.copyFile(fromAccessor, toAccessor, fromRecord, handler);
result.forward = true;
this.debug(fromAccessor, toAccessor, "file[from,to (to < from) => +to]", fullPath);
}
catch (e) {
if (e instanceof kura_1.NotFoundError) {
try {
await this.copyFile(toAccessor, fromAccessor, toRecord, handler);
result.backward = true;
this.debug(fromAccessor, toAccessor, "file[from,to (!-from) => +from]", fullPath);
}
catch (e2) {
if (e2 instanceof kura_1.NotFoundError) {
await this.deleteEntry(fromAccessor, fromRecord, handler);
await this.deleteEntry(toAccessor, toRecord, handler);
result.forward = true;
result.backward = true;
this.debug(fromAccessor, toAccessor, "file[from,to (!-from,!-to) => -from,-to]", fullPath);
}
else {
throw e2;
}
}
}
else {
throw e;
}
}
}
else if (fromModified < toModified) {
try {
await this.copyFile(toAccessor, fromAccessor, toRecord, handler);
result.backward = true;
this.debug(fromAccessor, toAccessor, "file[from,to (from < to) => +from]", fullPath);
}
catch (e) {
if (e instanceof kura_1.NotFoundError) {
try {
await this.copyFile(fromAccessor, toAccessor, toRecord, handler);
result.forward = true;
this.debug(fromAccessor, toAccessor, "file[from,to (!-to) => +to]", fullPath);
}
catch (e2) {
if (e2 instanceof kura_1.NotFoundError) {
await this.deleteEntry(fromAccessor, fromRecord, handler);
await this.deleteEntry(toAccessor, toRecord, handler);
result.forward = true;
result.backward = true;
this.debug(fromAccessor, toAccessor, "file[from,to (!-from,!-to) => -from,-to]", fullPath);
}
else {
throw e2;
}
}
}
else {
throw e;
}
}
}
else {
this.debug(fromAccessor, toAccessor, "file[from,to (from == to) =>]", fullPath);
}
}
}
else {
if (fromDeleted != null && toDeleted == null) {
if (fromDeleted < toModified) {
await this.makeDirectory(fromAccessor, toRecord);
await this.synchronizeChildren(toAccessor, fromAccessor, fullPath, true, notifier, handler);
result.backward = true;
this.debug(fromAccessor, toAccessor, "dir[-from,to (-from < to) => +from]", fullPath);
}
else {
if (toModified !== Synchronizer.NOT_EXISTS) {
await this.deleteEntry(toAccessor, toRecord, handler);
result.forward = true;
this.debug(fromAccessor, toAccessor, "dir[-from,to (to < -from) => -to]", fullPath);
}
else {
if (fromAccessor === this.localAccessor) {
await this.truncateRecord(toAccessor, fullPath);
}
this.debug(fromAccessor, toAccessor, "dir[-from,?to =>]", fullPath);
}
}
}
else if (fromDeleted == null && toDeleted != null) {
if (toDeleted < fromModified) {
await this.makeDirectory(toAccessor, fromRecord);
await this.synchronizeChildren(fromAccessor, toAccessor, fullPath, true, notifier, handler);
result.forward = true;
this.debug(fromAccessor, toAccessor, "dir[from,-to (-to < from) => +to]", fullPath);
}
else {
if (fromModified !== Synchronizer.NOT_EXISTS) {
await this.deleteEntry(fromAccessor, fromRecord, handler);
result.backward = true;
this.debug(toAccessor, fromAccessor, "dir[from,-to (from < -to) => -from]", fullPath);
}
else {
if (toAccessor === this.localAccessor) {
await this.truncateRecord(toAccessor, fullPath);
}
this.debug(toAccessor, fromAccessor, "dir[?from,-to =>]", fullPath);
}
}
}
else if (fromDeleted == null && toDeleted == null) {
if (fromModified === Synchronizer.NOT_EXISTS) {
await this.makeDirectory(fromAccessor, toRecord);
result.backward = true;
this.debug(fromAccessor, toAccessor, "dir[?from,to => +from]", fullPath);
}
else if (toModified === Synchronizer.NOT_EXISTS) {
await this.makeDirectory(toAccessor, fromRecord);
result.forward = true;
this.debug(fromAccessor, toAccessor, "dir[from,?to => +to]", fullPath);
}
else if (toModified < fromModified) {
await fromAccessor.saveRecord(fullPath, toRecord);
result.backward = true;
this.debug(fromAccessor, toAccessor, "dir[from,to (to < from) => +from]", fullPath);
}
else if (fromModified < toModified) {
await toAccessor.saveRecord(fullPath, fromRecord);
result.forward = true;
this.debug(fromAccessor, toAccessor, "dir[from,to (from < to) => +to]", fullPath);
}
else {
this.debug(fromAccessor, toAccessor, "dir[from,to (from == to) =>]", fullPath);
}
if (recursively) {
await this.synchronizeChildren(fromAccessor, toAccessor, fullPath, recursively, notifier, handler);
}
}
}
}
catch (e) {
result.errors.push(e);
this.warn(fromAccessor, toAccessor, name, e);
}
return result;
}
toFileSystemObject(record) {
const obj = {
fullPath: record.fullPath,
name: record.name,
lastModified: record.modified,
size: record.size,
};
return obj;
}
async truncateRecord(accessor, fullPath) {
const indexPath = await accessor.createIndexPath(fullPath, false);
try {
await accessor.doGetObject(fullPath, true);
await accessor.doDelete(indexPath, true);
}
catch (e) {
if (e instanceof kura_1.NotFoundError) {
console.info(e, fullPath, "truncateRecord");
}
else {
console.warn(e, fullPath, "truncateRecord");
}
}
}
warn(fromAccessor, toAccessor, path, e) {
if (!this.options.verbose) {
return;
}
const message = e ? JSON.stringify(e) : "";
console.warn(`${fromAccessor.name} => ${toAccessor.name}: ${path}\n` + message);
}
}
exports.Synchronizer = Synchronizer;
Synchronizer.NOT_EXISTS = 0;
//# sourceMappingURL=Synchronizer.js.map