federer
Version:
Experiments in asynchronous federated learning and decentralized learning
78 lines • 2.93 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ensureDataDownloaded = exports.getFilepaths = void 0;
const tslib_1 = require("tslib");
const path = tslib_1.__importStar(require("path"));
const fs = tslib_1.__importStar(require("fs"));
const node_fetch_1 = tslib_1.__importDefault(require("node-fetch"));
const node_gzip_1 = require("node-gzip");
const common_1 = require("../../../../common");
const fileNames = {
train: {
items: "train-images-idx3-ubyte",
labels: "train-labels-idx1-ubyte",
},
test: {
items: "t10k-images-idx3-ubyte",
labels: "t10k-labels-idx1-ubyte",
},
};
/**
* Returns the filepaths of the files that make up a dataset.
* To ensure that these files exist on disk, call `ensureDataDownloaded`.
*
* @param dataset Name of the dataset of which to get the filepaths for
* @returns Paths to the files that make up `dataset`
*/
function getFilepaths(dataset) {
return {
train: {
items: getFilepath(dataset, fileNames.train.items),
labels: getFilepath(dataset, fileNames.train.labels),
},
test: {
items: getFilepath(dataset, fileNames.test.items),
labels: getFilepath(dataset, fileNames.test.labels),
},
};
}
exports.getFilepaths = getFilepaths;
function getFilepath(dataset, filename) {
return path.join(common_1.absolutePath.data.raw(dataset), filename);
}
function getUrl(dataset, filename) {
switch (dataset) {
case "mnist":
return `https://storage.googleapis.com/cvdf-datasets/mnist/${filename}.gz`;
case "fashion-mnist":
return `http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/${filename}.gz`;
}
}
async function ensureDownloaded(url, destination) {
common_1.mkdirp(path.dirname(destination));
if (!fs.existsSync(destination)) {
// We disable the no-console rule here as an exception, because this
// function may be called by scripts, and it's a little overkill to create a
// logger for those.
// eslint-disable-next-line no-console
console.log(`Downloading ${url} to ${destination}`);
const response = await node_fetch_1.default(url);
const unzipped = await node_gzip_1.ungzip(await response.buffer());
return fs.promises.writeFile(destination, unzipped);
}
}
/**
* Ensure that the data for the MNIST dataset `dataset` is downloaded; if it is
* not available on disk, download it.
*/
async function ensureDataDownloaded(dataset) {
const files = [
fileNames.test.items,
fileNames.test.labels,
fileNames.train.items,
fileNames.train.labels,
];
await Promise.all(files.map((file) => ensureDownloaded(getUrl(dataset, file), getFilepath(dataset, file))));
}
exports.ensureDataDownloaded = ensureDataDownloaded;
//# sourceMappingURL=download.js.map