federer/dist/src/experiments/leaf/coordinator/read-json.d.ts

Experiments in asynchronous federated learning and decentralized learning

import * as tf from "@tensorflow/tfjs-node";
import { DataSubsetFilepaths, DataSubset } from "../../../common";
import { Environment } from "../../../coordinator/cli";
import { DatasetName } from "./LeafCoordinator";
export declare const NUM_LETTERS: number;
export interface LeafOutputPaths {
    trainpaths: DataSubsetFilepaths[];
    testpaths: DataSubsetFilepaths;
}
export declare function processRawData(dataset: DatasetName, environment: Environment, numberClients: number, numberLabelClasses: number): Promise<LeafOutputPaths>;
/**
 * Create shards for test and train datasets
 *
 * @param filepaths Holds file paths to both the train and test JSON files
 * @param environment Depending on the environment, data files will be read
 * either from local files, or from S3.
 * @param dataset The Leaf dataset to be used
 * @param numberLabelClasses Number of unique labels
 * @param train True if creating train shards
 * @param numberClients How many shards will be used for training.
 * @returns File paths to the train and test shards
 */
export declare function processRawDataFromFilepaths(filepaths: string, environment: Environment, dataset: DatasetName, numberLabelClasses: number, train: boolean, numberClients?: number): Promise<DataSubsetFilepaths[]>;
/**
 * Split the corresponding JSON file into shards
 *
 * @param file The path to train/test JSON file
 * @param environment Depending on the environment, data files will be read
 * either from local files, or from S3.
 * @param dataset The Leaf dataset to be used
 * @param numberLabelClasses Number of unique labels
 * @param numberClients How much of the total number
 * of shards will be used for training. This is
 * undefined if creating test shards
 * @returns An array of tensors grouped into subsets
 */
export declare function readRawDataFile(file: string, environment: Environment, dataset: DatasetName, numberLabelClasses: number, numberClients?: number): Promise<DataSubset[]>;
interface LeafInputPaths {
    train: string;
    test: string;
}
export declare function getFilepaths(dataset: string): LeafInputPaths;
declare type Shard = LeafJson["user_data"][string];
export declare function convertItems(array: any[], dataset: DatasetName): tf.Tensor;
export declare function convertLabels(array: any[], dataset: DatasetName, numberLabelClasses: number): tf.Tensor;
export declare function readLeafJson(file: string, environment: Environment): Promise<LeafJson>;
export declare function sample<T>(items: T[], samples: number): T[];
export declare function getFullData(items: Shard[]): Shard;
declare type LeafJsonType<Item, Label> = {
    users: string[];
    num_samples: number[];
    user_data: {
        [username: string]: {
            x: Item[];
            y: Label[];
        };
    };
};
declare type SyntheticJson = LeafJsonType<number[], number>;
declare type ShakespeareJson = LeafJsonType<string, string>;
declare type LeafJson = SyntheticJson | ShakespeareJson;
export declare function encodeSentence(sentence: string): number[];
export declare function encodeChar(char: string): number;
export declare function oneHotEncode(char: string): number[];
export {};
//# sourceMappingURL=read-json.d.ts.map