valor-client
Version:
A TypeScript API client for Valor.
724 lines (722 loc) • 30.9 kB
JavaScript
"use strict";
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (g && (g = 0, op[0] && (_ = 0)), _) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ValorClient = void 0;
var axios_1 = require("axios");
/**
* Checks if value conforms to the GeoJSON specification.
*
* @param value The value to type check.
* @returns A boolean result.
*/
function isGeoJSONObject(value) {
var geoJSONTypes = [
'point',
'linestring',
'polygon',
'multipoint',
'multilinestring',
'multipolygon'
];
return (typeof value === 'object' &&
value !== null &&
'type' in value &&
geoJSONTypes.includes(value.type.toLowerCase()));
}
/**
* Encodes metadata into the Valor API format.
*
* @param input An object containing metadata.
* @returns The encoded object.
*/
function encodeMetadata(input) {
var output = {};
for (var key in input) {
var value = input[key];
var valueType = void 0;
if (value instanceof Date) {
valueType = 'datetime';
output[key] = { type: valueType, value: value.toISOString() };
}
else if (isGeoJSONObject(value)) {
valueType = 'geojson';
output[key] = { type: valueType, value: value };
}
else if (typeof value === 'string' ||
typeof value === 'number' ||
typeof value === 'boolean') {
output[key] = value;
}
else {
console.warn("Unknown type for key \"".concat(key, "\"."));
output[key] = { type: typeof value, value: value };
}
}
return output;
}
/**
* Decodes metadata from the Valor API format.
*
* @param input An encoded Valor metadata object.
* @returns The decoded object.
*/
function decodeMetadata(input) {
var output = {};
for (var key in input) {
var item = input[key];
if (typeof item == 'object') {
var type = item.type, value = item.value;
switch (type.toLowerCase()) {
case 'datetime':
case 'date':
case 'time':
output[key] = new Date(value);
break;
case 'geojson':
output[key] = value;
break;
default:
console.warn("Unknown type for key \"".concat(key, "\"."));
output[key] = value;
break;
}
}
else {
output[key] = item;
}
}
return output;
}
var metadataDictToFilter = function (name, input) {
var args = Object.entries(input).map(function (_a) {
var key = _a[0], value = _a[1];
return ({
op: "eq",
lhs: {
name: name,
key: key
},
rhs: {
type: typeof value === 'string' ? 'string' : 'number',
value: value
}
});
});
return args.length === 1 ? args[0] : { op: "and", args: args };
};
var ValorClient = /** @class */ (function () {
/**
*
* @param baseURL - The base URL of the Valor server to connect to.
*/
function ValorClient(baseURL) {
this.client = axios_1.default.create({
baseURL: baseURL,
headers: {
'Content-Type': 'application/json'
}
});
}
/**
* Fetches datasets matching the filters defined by queryParams. This is private
* because we define higher-level methods that use this.
*
* @param filters An object containing a filter.
*
* @returns {Promise<Dataset[]>}
*
*/
ValorClient.prototype.getDatasets = function (filters) {
return __awaiter(this, void 0, void 0, function () {
var response, datasets, index, length_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.post('/datasets/filter', filters)];
case 1:
response = _a.sent();
datasets = response.data;
for (index = 0, length_1 = datasets.length; index < length_1; ++index) {
datasets[index].metadata = decodeMetadata(datasets[index].metadata);
}
return [2 /*return*/, datasets];
}
});
});
};
/**
* Fetches all datasets
*
* @returns {Promise<Dataset[]>}
*/
ValorClient.prototype.getAllDatasets = function () {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.getDatasets({})];
});
});
};
/**
* Fetches datasets matching a metadata object
*
* @param {{[key: string]: string | number}} metadata A metadata object to filter datasets by.
*
* @returns {Promise<Dataset[]>}
*
* @example
* const client = new ValorClient('http://localhost:8000/');
* client.getDatasetsByMetadata({ some_key: some_value }) // returns all datasets that have a metadata field `some_key` with value `some_value`
*
*/
ValorClient.prototype.getDatasetsByMetadata = function (metadata) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.getDatasets({ datasets: metadataDictToFilter("dataset.metadata", metadata) })];
});
});
};
/**
* Fetches a dataset given its name
*
* @param name name of the dataset
*
* @returns {Promise<Dataset>}
*/
ValorClient.prototype.getDatasetByName = function (name) {
return __awaiter(this, void 0, void 0, function () {
var response;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.get("/datasets/".concat(name))];
case 1:
response = _a.sent();
response.data.metadata = decodeMetadata(response.data.metadata);
return [2 /*return*/, response.data];
}
});
});
};
/**
* Creates a new dataset
*
* @param name name of the dataset
* @param metadata metadata of the dataset
*
* @returns {Promise<void>}
*/
ValorClient.prototype.createDataset = function (name, metadata) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
metadata = encodeMetadata(metadata);
return [4 /*yield*/, this.client.post('/datasets', { name: name, metadata: metadata })];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
};
/**
* Finalizes a dataset (which is necessary to run an evaluation)
*
* @param name name of the dataset to finalize
*
* @returns {Promise<void>}
*/
ValorClient.prototype.finalizeDataset = function (name) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.put("/datasets/".concat(name, "/finalize"))];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
};
/**
* Deletes a dataset
*
* @param name name of the dataset to delete
*
* @returns {Promise<void>}
*/
ValorClient.prototype.deleteDataset = function (name) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.delete("/datasets/".concat(name))];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
};
/**
* Fetches models matching the filters defined by queryParams. This is
* private because we define higher-level methods that use this.
*
* @param filters An object containing query parameters to filter models by.
*
* @returns {Promise<Model[]>}
*/
ValorClient.prototype.getModels = function (filters) {
return __awaiter(this, void 0, void 0, function () {
var response, models, index, length_2;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.post('/models/filter', filters)];
case 1:
response = _a.sent();
models = response.data;
for (index = 0, length_2 = models.length; index < length_2; ++index) {
models[index].metadata = decodeMetadata(models[index].metadata);
}
return [2 /*return*/, models];
}
});
});
};
/**
* Fetches all models
*
* @returns {Promise<Model[]>}
*/
ValorClient.prototype.getAllModels = function () {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.getModels({})];
});
});
};
/**
* Fetches models matching a metadata object
*
* @param {{[key: string]: string | number}} metadata A metadata object to filter models by.
*
* @returns {Promise<Model[]>}
*
* @example
* const client = new ValorClient('http://localhost:8000/');
* client.getModelsByMetadata({ some_key: some_value }) // returns all models that have a metadata field `some_key` with value `some_value`
*/
ValorClient.prototype.getModelsByMetadata = function (metadata) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.getModels({ models: metadataDictToFilter("model.metadata", metadata) })];
});
});
};
/**
* Fetches a model given its name
*
* @param name name of the model
*
* @returns {Promise<Model>}
*/
ValorClient.prototype.getModelByName = function (name) {
return __awaiter(this, void 0, void 0, function () {
var response;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.get("/models/".concat(name))];
case 1:
response = _a.sent();
response.data.metadata = decodeMetadata(response.data.metadata);
return [2 /*return*/, response.data];
}
});
});
};
/**
* Creates a new model
*
* @param name name of the model
* @param metadata metadata of the model
*
* @returns {Promise<void>}
*/
ValorClient.prototype.createModel = function (name, metadata) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
metadata = encodeMetadata(metadata);
return [4 /*yield*/, this.client.post('/models', { name: name, metadata: metadata })];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
};
/**
* Deletes a model
*
* @param name name of the model to delete
*
* @returns {Promise<void>}
*/
ValorClient.prototype.deleteModel = function (name) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.delete("/models/".concat(name))];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
};
/**
* Takes data from the backend response and converts it to an Evaluation object
* by converting the datetime string to a `Date` object and replacing -1 metric values with
* `null`.
*/
ValorClient.prototype.unmarshalEvaluation = function (evaluation) {
var updatedMetrics = evaluation.metrics.map(function (metric) { return (__assign(__assign({}, metric), { value: metric.value === -1 ? null : metric.value })); });
return __assign(__assign({}, evaluation), { metrics: updatedMetrics, created_at: new Date(evaluation.created_at) });
};
/**
* Creates a new evaluation or gets an existing one if an evaluation with the
* same parameters already exists.
*
* @param model name of the model
* @param dataset name of the dataset
* @param taskType type of task
* @param [metrics_to_return] The list of metrics to compute, store, and return to the user.
* @param [iouThresholdsToCompute] list of floats describing which Intersection over Unions (IoUs) to use when calculating metrics (i.e., mAP)
* @param [iouThresholdsToReturn] list of floats describing which Intersection over Union (IoUs) thresholds to calculate a metric for. Must be a subset of `iou_thresholds_to_compute`
* @param [labelMap] mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models
* @param [recallScoreThreshold] confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall
* @param [prCurveIouThreshold] the IOU threshold to use when calculating precision-recall curves for object detection tasks. Defaults to 0.5.
* @param [prCurveMaxExamples] the maximum number of datum examples to store for each error type when calculating PR curves.
*
* @returns {Promise<Evaluation>}
*/
ValorClient.prototype.createOrGetEvaluation = function (model, dataset, taskType, metrics_to_return, iouThresholdsToCompute, iouThresholdsToReturn, labelMap, recallScoreThreshold, prCurveIouThreshold, prCurveMaxExamples) {
return __awaiter(this, void 0, void 0, function () {
var response;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.post('/evaluations', {
dataset_names: [dataset],
model_names: [model],
filters: {},
parameters: {
task_type: taskType,
iou_thresholds_to_compute: iouThresholdsToCompute,
iou_thresholds_to_return: iouThresholdsToReturn,
label_map: labelMap,
recall_score_threshold: recallScoreThreshold,
metrics_to_return: metrics_to_return,
pr_curve_iou_threshold: prCurveIouThreshold,
pr_curve_max_examples: prCurveMaxExamples
},
})];
case 1:
response = _a.sent();
return [2 /*return*/, this.unmarshalEvaluation(response.data[0])];
}
});
});
};
/**
* Creates new evaluations given a list of models, or gets existing ones if evaluations with the
* same parameters already exists.
*
* @param models names of the models
* @param dataset name of the dataset
* @param taskType type of task
* @param [metrics_to_return] The list of metrics to compute, store, and return to the user.
* @param [iouThresholdsToCompute] list of floats describing which Intersection over Unions (IoUs) to use when calculating metrics (i.e., mAP)
* @param [iouThresholdsToReturn] list of floats describing which Intersection over Union (IoUs) thresholds to calculate a metric for. Must be a subset of `iou_thresholds_to_compute`
* @param [labelMap] mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models
* @param [recallScoreThreshold] confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall
* @param [prCurveIouThreshold] the IOU threshold to use when calculating precision-recall curves for object detection tasks. Defaults to 0.5
* @param [prCurveMaxExamples] the maximum number of datum examples to store for each error type when calculating PR curves.
*
* @returns {Promise<Evaluation[]>}
*/
ValorClient.prototype.bulkCreateOrGetEvaluations = function (models, dataset, taskType, metrics_to_return, iouThresholdsToCompute, iouThresholdsToReturn, labelMap, recallScoreThreshold, prCurveIouThreshold, prCurveMaxExamples) {
return __awaiter(this, void 0, void 0, function () {
var response;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.post('/evaluations', {
dataset_names: [dataset],
model_names: models,
filters: {},
parameters: {
task_type: taskType,
metrics_to_return: metrics_to_return,
iou_thresholds_to_compute: iouThresholdsToCompute,
iou_thresholds_to_return: iouThresholdsToReturn,
label_map: labelMap,
recall_score_threshold: recallScoreThreshold,
pr_curve_iou_threshold: prCurveIouThreshold,
pr_curve_max_examples: prCurveMaxExamples
},
})];
case 1:
response = _a.sent();
return [2 /*return*/, response.data.map(this.unmarshalEvaluation)];
}
});
});
};
/**
* Fetches evaluations matching the filters defined by queryParams. This is
* private because we define higher-level methods that use this.
*
* @param queryParams An object containing query parameters to filter evaluations by.
*
* @returns {Promise<Evaluation[]>}
*/
ValorClient.prototype.getEvaluations = function (queryParams) {
return __awaiter(this, void 0, void 0, function () {
var response;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.client.get('/evaluations', { params: queryParams })];
case 1:
response = _a.sent();
return [2 /*return*/, response.data.map(this.unmarshalEvaluation)];
}
});
});
};
/**
* Fetches an evaluation by id
*
* @param id id of the evaluation
* @param offset The start index of the evaluations to return. Used for pagination.
* @param limit The number of evaluations to return. Used for pagination.
* @param metricsToSortBy A map of metrics to sort the evaluations by.
*
* @returns {Promise<Evaluation>}
*/
ValorClient.prototype.getEvaluationById = function (id, offset, limit, metricsToSortBy) {
return __awaiter(this, void 0, void 0, function () {
var evaluations;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.getEvaluations({
evaluation_ids: id,
offset: offset,
limit: limit,
metrics_to_sort_by: metricsToSortBy != null ? JSON.stringify(metricsToSortBy) : null
})];
case 1:
evaluations = _a.sent();
return [2 /*return*/, evaluations[0]];
}
});
});
};
/**
* Bulk fetches evaluation by array of ids
*
* @param id id of the evaluation
* @param offset The start index of the evaluations to return. Used for pagination.
* @param limit The number of evaluations to return. Used for pagination.
* @param metricsToSortBy A map of metrics to sort the evaluations by.
*
* @returns {Promise<Evaluation[]>}
*/
ValorClient.prototype.getEvaluationsByIds = function (ids, offset, limit, metricsToSortBy) {
return __awaiter(this, void 0, void 0, function () {
var evaluations;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.getEvaluations({
evaluation_ids: ids.map(function (id) { return id.toString(); }).join(','),
offset: offset,
limit: limit,
metrics_to_sort_by: metricsToSortBy != null ? JSON.stringify(metricsToSortBy) : null
})];
case 1:
evaluations = _a.sent();
return [2 /*return*/, evaluations];
}
});
});
};
/**
* Fetches all evaluations associated to given models
*
* @param modelNames names of the models
* @param offset The start index of the evaluations to return. Used for pagination.
* @param limit The number of evaluations to return. Used for pagination.
* @param metricsToSortBy A map of metrics to sort the evaluations by.
*
* @returns {Promise<Evaluation[]>}
*/
ValorClient.prototype.getEvaluationsByModelNames = function (modelNames, offset, limit, metricsToSortBy) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
// turn modelNames into a comma-separated string
return [2 /*return*/, this.getEvaluations({
models: modelNames.join(','),
offset: offset,
limit: limit,
metrics_to_sort_by: metricsToSortBy != null ? JSON.stringify(metricsToSortBy) : null
})];
});
});
};
/**
* Fetches all evaluations associated to given datasets
*
* @param datasetNames names of the datasets
* @param offset The start index of the evaluations to return. Used for pagination.
* @param limit The number of evaluations to return. Used for pagination.
* @param metricsToSortBy A map of metrics to sort the evaluations by.
*
* @returns {Promise<Evaluation[]>}
*/
ValorClient.prototype.getEvaluationsByDatasetNames = function (datasetNames, offset, limit, metricsToSortBy) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.getEvaluations({
datasets: datasetNames.join(','),
offset: offset,
limit: limit,
metrics_to_sort_by: metricsToSortBy != null ? JSON.stringify(metricsToSortBy) : null
})];
});
});
};
/**
* Fetches all evaluations associated to given models and dataset names
*
* @param modelNames names of the models
* @param datasetNames names of the datasets
* @param offset The start index of the evaluations to return. Used for pagination.
* @param limit The number of evaluations to return. Used for pagination.
* @param metricsToSortBy A map of metrics to sort the evaluations by.
*
* @returns {Promise<Evaluation[]>}
*/
ValorClient.prototype.getEvaluationsByModelNamesAndDatasetNames = function (modelNames, datasetNames, offset, limit, metricsToSortBy) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.getEvaluations({
models: modelNames.join(','),
datasets: datasetNames.join(','),
offset: offset,
limit: limit,
metrics_to_sort_by: metricsToSortBy != null ? JSON.stringify(metricsToSortBy) : null
})];
});
});
};
/**
* Adds ground truth annotations to a dataset
*
* @param datasetName name of the dataset
* @param datum valor datum
* @param annotations valor annotations
*
* @returns {Promise<void>}
*/
ValorClient.prototype.addGroundTruth = function (datasetName, datum, annotations) {
return __awaiter(this, void 0, void 0, function () {
var index, length_3;
return __generator(this, function (_a) {
datum.metadata = encodeMetadata(datum.metadata);
for (index = 0, length_3 = annotations.length; index < length_3; ++index) {
annotations[index].metadata = encodeMetadata(annotations[index].metadata);
}
return [2 /*return*/, this.client.post('/groundtruths', [
{
dataset_name: datasetName,
datum: datum,
annotations: annotations
}
])];
});
});
};
/**
* Adds predictions from a model
*
* @param datasetName name of the dataset
* @param modelName name of the model
* @param datum valor datum
* @param annotations valor annotations
*
* @returns {Promise<void>}
*/
ValorClient.prototype.addPredictions = function (datasetName, modelName, datum, annotations) {
return __awaiter(this, void 0, void 0, function () {
var index, length_4;
return __generator(this, function (_a) {
datum.metadata = encodeMetadata(datum.metadata);
for (index = 0, length_4 = annotations.length; index < length_4; ++index) {
annotations[index].metadata = encodeMetadata(annotations[index].metadata);
}
return [2 /*return*/, this.client.post('/predictions', [
{
dataset_name: datasetName,
model_name: modelName,
datum: datum,
annotations: annotations
}
])];
});
});
};
return ValorClient;
}());
exports.ValorClient = ValorClient;