federer
Version:
Experiments in asynchronous federated learning and decentralized learning
122 lines • 4.82 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.RelativeDataPaths = exports.AbsoluteDataPaths = void 0;
const tslib_1 = require("tslib");
const assert = require("assert");
const fs = tslib_1.__importStar(require("fs"));
const path = tslib_1.__importStar(require("path"));
const common_1 = require("../../common");
/**
* We cache the results of preprocessing in files.
*
* This class represents the contents of the JSON file. The JSON file should be
* saved with relative paths, so that it is portable (i.e. can be generated
* locally, but used in a Docker container or another machine).
*/
class PreprocessedDataPaths {
constructor(json) {
this.json = json;
const isAbsolute = this.checkAbsolute();
const isRelative = this.checkRelative();
assert(isAbsolute || isRelative, `Object contains mixed relative and absolute paths: ${this.getJSON()}`);
assert(isAbsolute !== isRelative, `Cannot be both absolute and relative`);
}
checkAbsolute() {
return this.every((file) => path.isAbsolute(file));
}
checkRelative() {
return this.every((file) => !path.isAbsolute(file));
}
/**
* Returns JSON string representation of the object.
* @param pretty Whether to return pretty-printed JSON (indented)
*/
getJSON(pretty = true) {
if (pretty) {
return JSON.stringify(this.json, null, " ");
}
else {
return JSON.stringify(this.json);
}
}
/**
* Map values in this object to new values, using the given mapping function.
*
* @param fn Mapping function
* @returns Object with values that are the result of applying `fn` to the
* current object's values.
*/
mapValues(fn) {
const datasubsetFn = (paths) => ({
items: fn(paths.items),
labels: fn(paths.labels),
});
return {
testSetFiles: datasubsetFn(this.json.testSetFiles),
clientTrainFiles: this.json.clientTrainFiles.map((files) => datasubsetFn(files)),
dataDistribution: {
distributionMatrix: fn(this.json.dataDistribution.distributionMatrix),
},
};
}
/**
* Returns whether a given predicate holds for all values in this object.
*
* @param predicate Function returning a boolean indicating whether the
* condition to check holds.
*
* @returns `true` if `predicate` returns `true` on all values in the object,
* `false` if it returns `false` on at least one value.
*/
every(predicate) {
const getFiles = (paths) => [
paths.items,
paths.labels,
];
const files = [
...getFiles(this.json.testSetFiles),
...this.json.clientTrainFiles.flatMap((paths) => getFiles(paths)),
this.json.dataDistribution.distributionMatrix,
];
return files.every((file) => predicate(file));
}
}
class AbsoluteDataPaths extends PreprocessedDataPaths {
constructor(json) {
super(json);
assert(this.checkAbsolute());
}
toRelative(parentDir) {
assert(path.isAbsolute(parentDir));
return new RelativeDataPaths(this.mapValues((absolute) => path.relative(parentDir, absolute)), parentDir);
}
}
exports.AbsoluteDataPaths = AbsoluteDataPaths;
class RelativeDataPaths extends PreprocessedDataPaths {
constructor(json, rootDir) {
super(json);
this.rootDir = rootDir;
assert(path.isAbsolute(rootDir));
assert(this.checkRelative(), "All paths must be relative");
}
toAbsolute() {
return new AbsoluteDataPaths(this.mapValues((relative) => path.resolve(this.rootDir, relative)));
}
static async load(filepath) {
assert.strictEqual(path.extname(filepath), ".json", `Expected to read from a .json file, but got filepath ${filepath}`);
const buffer = await fs.promises.readFile(filepath);
const json = JSON.parse(buffer.toString());
return new RelativeDataPaths(json, path.dirname(filepath));
}
async save(filepath) {
if (!path.isAbsolute(filepath)) {
filepath = path.join(this.rootDir, filepath);
}
assert.strictEqual(path.dirname(filepath), this.rootDir, `Expected to save to the root directory ${this.rootDir} but got filepath ${filepath}`);
assert(common_1.isParentDirectory(this.rootDir, filepath)); // sanity check
assert.strictEqual(path.extname(filepath), ".json", `Expected to write to a .json file, but got filepath ${filepath}`);
return fs.promises.writeFile(filepath, this.getJSON());
}
}
exports.RelativeDataPaths = RelativeDataPaths;
//# sourceMappingURL=PreprocessedDataPaths.js.map