valor-client/dist/ValorClient.d.ts

A TypeScript API client for Valor.

export type TaskType = 'skip' | 'empty' | 'classification' | 'object-detection' | 'semantic-segmentation' | 'embedding';
export type Label = {
    key: string;
    value: string;
    score?: number;
};
export type Dataset = {
    name: string;
    metadata: Partial<Record<string, any>>;
};
export type Model = {
    name: string;
    metadata: Partial<Record<string, any>>;
};
export type Datum = {
    uid: string;
    metadata: Partial<Record<string, any>>;
};
export type Annotation = {
    metadata: Partial<Record<string, any>>;
    labels: Label[];
    bounding_box?: number[][][];
    polygon?: number[][][];
    raster?: object;
    embedding?: number[];
    is_instance?: boolean;
};
export type Metric = {
    type: string;
    parameters?: Partial<Record<string, any>>;
    value: number | any;
    label?: Label;
};
export type Evaluation = {
    id: number;
    dataset_names: string[];
    model_name: string;
    filters: any;
    parameters: {
        task_type: TaskType;
        object: any;
    };
    status: 'pending' | 'running' | 'done' | 'failed' | 'deleting';
    metrics: Metric[];
    confusion_matrices: any[];
    created_at: Date;
};
export declare class ValorClient {
    private client;
    /**
     *
     * @param baseURL - The base URL of the Valor server to connect to.
     */
    constructor(baseURL: string);
    /**
     * Fetches datasets matching the filters defined by queryParams. This is private
     * because we define higher-level methods that use this.
     *
     * @param filters An object containing a filter.
     *
     * @returns {Promise<Dataset[]>}
     *
     */
    private getDatasets;
    /**
     * Fetches all datasets
     *
     * @returns {Promise<Dataset[]>}
     */
    getAllDatasets(): Promise<Dataset[]>;
    /**
     * Fetches datasets matching a metadata object
     *
     * @param {{[key: string]: string | number}} metadata A metadata object to filter datasets by.
     *
     * @returns {Promise<Dataset[]>}
     *
     * @example
     * const client = new ValorClient('http://localhost:8000/');
     * client.getDatasetsByMetadata({ some_key: some_value }) // returns all datasets that have a metadata field `some_key` with value `some_value`
     *
     */
    getDatasetsByMetadata(metadata: {
        [key: string]: string | number;
    }): Promise<Dataset[]>;
    /**
     * Fetches a dataset given its name
     *
     * @param name name of the dataset
     *
     * @returns {Promise<Dataset>}
     */
    getDatasetByName(name: string): Promise<Dataset>;
    /**
     * Creates a new dataset
     *
     * @param name name of the dataset
     * @param metadata metadata of the dataset
     *
     * @returns {Promise<void>}
     */
    createDataset(name: string, metadata: object): Promise<void>;
    /**
     * Finalizes a dataset (which is necessary to run an evaluation)
     *
     * @param name name of the dataset to finalize
     *
     * @returns {Promise<void>}
     */
    finalizeDataset(name: string): Promise<void>;
    /**
     * Deletes a dataset
     *
     * @param name name of the dataset to delete
     *
     * @returns {Promise<void>}
     */
    deleteDataset(name: string): Promise<void>;
    /**
     * Fetches models matching the filters defined by queryParams. This is
     * private because we define higher-level methods that use this.
     *
     * @param filters An object containing query parameters to filter models by.
     *
     * @returns {Promise<Model[]>}
     */
    private getModels;
    /**
     * Fetches all models
     *
     * @returns {Promise<Model[]>}
     */
    getAllModels(): Promise<Model[]>;
    /**
     * Fetches models matching a metadata object
     *
     * @param {{[key: string]: string | number}} metadata A metadata object to filter models by.
     *
     * @returns {Promise<Model[]>}
     *
     * @example
     * const client = new ValorClient('http://localhost:8000/');
     * client.getModelsByMetadata({ some_key: some_value }) // returns all models that have a metadata field `some_key` with value `some_value`
     */
    getModelsByMetadata(metadata: {
        [key: string]: string | number;
    }): Promise<Model[]>;
    /**
     * Fetches a model given its name
     *
     * @param name name of the model
     *
     * @returns {Promise<Model>}
     */
    getModelByName(name: string): Promise<Model>;
    /**
     * Creates a new model
     *
     * @param name name of the model
     * @param metadata metadata of the model
     *
     * @returns {Promise<void>}
     */
    createModel(name: string, metadata: object): Promise<void>;
    /**
     * Deletes a model
     *
     * @param name name of the model to delete
     *
     * @returns {Promise<void>}
     */
    deleteModel(name: string): Promise<void>;
    /**
     * Takes data from the backend response and converts it to an Evaluation object
     * by converting the datetime string to a `Date` object and replacing -1 metric values with
     * `null`.
     */
    private unmarshalEvaluation;
    /**
     * Creates a new evaluation or gets an existing one if an evaluation with the
     * same parameters already exists.
     *
     * @param model name of the model
     * @param dataset name of the dataset
     * @param taskType type of task
     * @param [metrics_to_return] The list of metrics to compute, store, and return to the user.
     * @param [iouThresholdsToCompute] list of floats describing which Intersection over Unions (IoUs) to use when calculating metrics (i.e., mAP)
     * @param [iouThresholdsToReturn] list of floats describing which Intersection over Union (IoUs) thresholds to calculate a metric for. Must be a subset of `iou_thresholds_to_compute`
     * @param [labelMap] mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models
     * @param [recallScoreThreshold] confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall
     * @param [prCurveIouThreshold] the IOU threshold to use when calculating precision-recall curves for object detection tasks. Defaults to 0.5.
     * @param [prCurveMaxExamples] the maximum number of datum examples to store for each error type when calculating PR curves.
     *
     * @returns {Promise<Evaluation>}
     */
    createOrGetEvaluation(model: string, dataset: string, taskType: TaskType, metrics_to_return?: string[], iouThresholdsToCompute?: number[], iouThresholdsToReturn?: number[], labelMap?: number[][][], recallScoreThreshold?: number, prCurveIouThreshold?: number, prCurveMaxExamples?: number): Promise<Evaluation>;
    /**
     * Creates new evaluations given a list of models, or gets existing ones if evaluations with the
     * same parameters already exists.
     *
     * @param models names of the models
     * @param dataset name of the dataset
     * @param taskType type of task
     * @param [metrics_to_return] The list of metrics to compute, store, and return to the user.
     * @param [iouThresholdsToCompute] list of floats describing which Intersection over Unions (IoUs) to use when calculating metrics (i.e., mAP)
     * @param [iouThresholdsToReturn] list of floats describing which Intersection over Union (IoUs) thresholds to calculate a metric for. Must be a subset of `iou_thresholds_to_compute`
     * @param [labelMap] mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models
     * @param [recallScoreThreshold] confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall
     * @param [prCurveIouThreshold] the IOU threshold to use when calculating precision-recall curves for object detection tasks. Defaults to 0.5
     * @param [prCurveMaxExamples] the maximum number of datum examples to store for each error type when calculating PR curves.
  
     *
     * @returns {Promise<Evaluation[]>}
     */
    bulkCreateOrGetEvaluations(models: string[], dataset: string, taskType: TaskType, metrics_to_return?: string[], iouThresholdsToCompute?: number[], iouThresholdsToReturn?: number[], labelMap?: any[][][], recallScoreThreshold?: number, prCurveIouThreshold?: number, prCurveMaxExamples?: number): Promise<Evaluation[]>;
    /**
     * Fetches evaluations matching the filters defined by queryParams. This is
     * private because we define higher-level methods that use this.
     *
     * @param queryParams An object containing query parameters to filter evaluations by.
     *
     * @returns {Promise<Evaluation[]>}
     */
    private getEvaluations;
    /**
     * Fetches an evaluation by id
     *
     * @param id id of the evaluation
     * @param offset The start index of the evaluations to return. Used for pagination.
     * @param limit The number of evaluations to return. Used for pagination.
     * @param metricsToSortBy A map of metrics to sort the evaluations by.
     *
     * @returns {Promise<Evaluation>}
     */
    getEvaluationById(id: number, offset?: number, limit?: number, metricsToSortBy?: {
        [key: string]: string | {
            [inner_key: string]: string;
        };
    }): Promise<Evaluation>;
    /**
     * Bulk fetches evaluation by array of ids
     *
     * @param id id of the evaluation
     * @param offset The start index of the evaluations to return. Used for pagination.
     * @param limit The number of evaluations to return. Used for pagination.
     * @param metricsToSortBy A map of metrics to sort the evaluations by.
     *
     * @returns {Promise<Evaluation[]>}
     */
    getEvaluationsByIds(ids: number[], offset?: number, limit?: number, metricsToSortBy?: {
        [key: string]: string | {
            [inner_key: string]: string;
        };
    }): Promise<Evaluation[]>;
    /**
     * Fetches all evaluations associated to given models
     *
     * @param modelNames names of the models
     * @param offset The start index of the evaluations to return. Used for pagination.
     * @param limit The number of evaluations to return. Used for pagination.
     * @param metricsToSortBy A map of metrics to sort the evaluations by.
     *
     * @returns {Promise<Evaluation[]>}
     */
    getEvaluationsByModelNames(modelNames: string[], offset?: number, limit?: number, metricsToSortBy?: {
        [key: string]: string | {
            [inner_key: string]: string;
        };
    }): Promise<Evaluation[]>;
    /**
     * Fetches all evaluations associated to given datasets
     *
     * @param datasetNames names of the datasets
     * @param offset The start index of the evaluations to return. Used for pagination.
     * @param limit The number of evaluations to return. Used for pagination.
     * @param metricsToSortBy A map of metrics to sort the evaluations by.
     *
     * @returns {Promise<Evaluation[]>}
     */
    getEvaluationsByDatasetNames(datasetNames: string[], offset?: number, limit?: number, metricsToSortBy?: {
        [key: string]: string | {
            [inner_key: string]: string;
        };
    }): Promise<Evaluation[]>;
    /**
     * Fetches all evaluations associated to given models and dataset names
     *
     * @param modelNames names of the models
     * @param datasetNames names of the datasets
     * @param offset The start index of the evaluations to return. Used for pagination.
     * @param limit The number of evaluations to return. Used for pagination.
     * @param metricsToSortBy A map of metrics to sort the evaluations by.
     *
     * @returns {Promise<Evaluation[]>}
     */
    getEvaluationsByModelNamesAndDatasetNames(modelNames: string[], datasetNames: string[], offset?: number, limit?: number, metricsToSortBy?: {
        [key: string]: string | {
            [inner_key: string]: string;
        };
    }): Promise<Evaluation[]>;
    /**
     * Adds ground truth annotations to a dataset
     *
     * @param datasetName name of the dataset
     * @param datum valor datum
     * @param annotations valor annotations
     *
     * @returns {Promise<void>}
     */
    addGroundTruth(datasetName: string, datum: Datum, annotations: Annotation[]): Promise<void>;
    /**
     * Adds predictions from a model
     *
     * @param datasetName name of the dataset
     * @param modelName name of the model
     * @param datum valor datum
     * @param annotations valor annotations
     *
     * @returns {Promise<void>}
     */
    addPredictions(datasetName: string, modelName: string, datum: Datum, annotations: Annotation[]): Promise<void>;
}