@future-agi/sdk
Version:
We help GenAI teams maintain high-accuracy for their Models in production.
280 lines • 10.4 kB
JavaScript
import { APIKeyAuth } from '../api/auth.js';
import { HttpMethod } from '../api/types.js';
import { Routes } from '../utils/routes.js';
import { SDKException, InvalidAuthError } from '../utils/errors.js';
export class Annotation extends APIKeyAuth {
/**
* SDK client for logging human annotations using flat DataFrame-style format.
*
*/
constructor(options = {}) {
super({
fiApiKey: options.fiApiKey,
fiSecretKey: options.fiSecretKey,
fiBaseUrl: options.fiBaseUrl,
timeout: options.timeout
});
}
async logAnnotations(records, options = {}) {
/**
* Log annotations using flat DataFrame-style format.
*
* Expected record format:
* - context.span_id: Span ID for the annotation
* - annotation.{name}.text: Text annotations
* - annotation.{name}.label: Categorical annotations
* - annotation.{name}.score: Numeric annotations
* - annotation.{name}.rating: Star ratings (1-5)
* - annotation.{name}.thumbs: Thumbs up/down (true/false)
* - annotation.notes: Optional notes text
*
* @param records Array of annotation records
* @param options.projectName Project name for label scoping
* @param options.timeout Request timeout
*
* @example
* ```typescript
* const records = [
* {
* 'context.span_id': 'span123',
* 'annotation.quality.text': 'good response',
* 'annotation.rating.rating': 4,
* 'annotation.helpful.thumbs': true,
* 'annotation.notes': 'Great response!'
* }
* ];
*
* const response = await client.logAnnotations(records, {
* projectName: 'My Project'
* });
* ```
*/
if (!Array.isArray(records)) {
throw new Error('Records must be an array');
}
// Convert flat records to nested backend format
const backendRecords = await this._convertRecordsToBackendFormat(records, options.projectName);
console.log(`Sending ${backendRecords.length} annotation records via bulk endpoint`);
const config = {
method: HttpMethod.POST,
url: `${this.baseUrl}/${Routes.BULK_ANNOTATION}`,
data: { records: backendRecords },
timeout: options.timeout,
};
try {
const response = await this.request(config);
return this._parseBulkAnnotationResponse(response.data);
}
catch (error) {
if (error.response?.status === 403) {
throw new InvalidAuthError();
}
throw new SDKException(error.response?.data?.message || 'Bulk annotation request failed');
}
}
async getLabels(options = {}) {
/**
* Fetch annotation labels available to the user.
*/
const params = {};
if (options.projectId) {
params.project_id = options.projectId;
}
const config = {
method: HttpMethod.GET,
url: `${this.baseUrl}/${Routes.GET_ANNOTATION_LABELS}`,
params,
timeout: options.timeout,
};
try {
const response = await this.request(config);
const data = response.data;
// Handle wrapped response
const labelsData = data.result || data;
return labelsData.map((item) => ({
id: item.id,
name: item.name,
type: item.type,
description: item.description,
settings: item.settings,
}));
}
catch (error) {
if (error.response?.status === 403) {
throw new InvalidAuthError();
}
throw new SDKException('Failed to fetch annotation labels');
}
}
async listProjects(options = {}) {
/**
* List available projects.
*/
const params = {
page_number: options.pageNumber || 0,
page_size: options.pageSize || 20,
};
if (options.projectType) {
params.project_type = options.projectType;
}
if (options.name) {
params.name = options.name;
}
const config = {
method: HttpMethod.GET,
url: `${this.baseUrl}/${Routes.LIST_PROJECTS}`,
params,
timeout: options.timeout,
};
try {
const response = await this.request(config);
const data = response.data;
// Handle wrapped response with metadata
let projectsData = data.result || data;
if (projectsData.table) {
projectsData = projectsData.table;
}
return projectsData.map((item) => ({
id: item.id,
name: item.name,
project_type: item.project_type,
created_at: item.created_at,
}));
}
catch (error) {
if (error.response?.status === 403) {
throw new InvalidAuthError();
}
throw new SDKException('Failed to list projects');
}
}
async _convertRecordsToBackendFormat(records, projectName) {
const backendRecords = [];
for (const record of records) {
const spanId = record['context.span_id'];
if (!spanId) {
continue;
}
const backendRecord = {
observation_span_id: spanId,
annotations: [],
notes: [],
};
// Process annotation columns
for (const [key, value] of Object.entries(record)) {
if (key.startsWith('annotation.') && key !== 'annotation.notes' && value != null) {
const annotation = await this._parseAnnotationColumn(key, value, projectName);
if (annotation) {
backendRecord.annotations.push(annotation);
}
}
}
// Process notes
if (record['annotation.notes']) {
backendRecord.notes.push({ text: String(record['annotation.notes']) });
}
if (backendRecord.annotations.length > 0 || backendRecord.notes.length > 0) {
backendRecords.push(backendRecord);
}
}
return backendRecords;
}
async _parseAnnotationColumn(column, value, projectName) {
// Format: annotation.{name}.{type}
const parts = column.split('.');
if (parts.length !== 3) {
return null;
}
const [, name, valueType] = parts;
// Get project-specific label ID
const labelId = await this._getLabelIdForNameAndType(name, valueType, projectName);
if (!labelId) {
throw new Error(`No annotation label found for name '${name}' and type '${valueType}' in project '${projectName}'`);
}
// Map column types to backend fields
switch (valueType) {
case 'text':
return {
annotation_label_id: labelId,
value: String(value),
};
case 'label':
return {
annotation_label_id: labelId,
value_str_list: Array.isArray(value)
? value.map(String)
: [String(value)],
};
case 'score':
return {
annotation_label_id: labelId,
value_float: Number(value),
};
case 'rating':
return {
annotation_label_id: labelId,
value_float: Number(value),
};
case 'thumbs':
return {
annotation_label_id: labelId,
value_bool: Boolean(value),
};
default:
return null;
}
}
async _getProjectId(projectName) {
const projects = await this.listProjects({ name: projectName });
if (projects.length === 0) {
throw new Error(`Project '${projectName}' not found`);
}
if (projects.length > 1) {
const projectList = projects.map(p => `${p.name} (id: ${p.id})`).join(', ');
throw new Error(`Multiple projects found for '${projectName}': ${projectList}`);
}
return projects[0].id;
}
async _getLabelIdForNameAndType(name, columnType, projectName) {
// Get project ID if project name provided
let projectId;
if (projectName) {
projectId = await this._getProjectId(projectName);
}
// Get labels (filtered by project if specified)
const labels = await this.getLabels({ projectId });
// Map column types to backend label types
const typeMapping = {
text: 'text',
label: 'categorical',
score: 'numeric',
rating: 'star',
thumbs: 'thumbs_up_down',
};
const expectedLabelType = typeMapping[columnType];
if (!expectedLabelType) {
return null;
}
// Find matching label
const matchingLabel = labels.find(label => label.name === name && label.type.toLowerCase() === expectedLabelType);
return matchingLabel?.id || null;
}
_parseBulkAnnotationResponse(data) {
// Handle wrapped response
if (data.result) {
data = data.result;
}
return {
message: data.message || 'Bulk annotation completed',
annotationsCreated: data.annotationsCreated || 0,
annotationsUpdated: data.annotationsUpdated || 0,
notesCreated: data.notesCreated || 0,
succeededCount: data.succeededCount || 0,
errorsCount: data.errorsCount || 0,
warningsCount: data.warningsCount || 0,
warnings: data.warnings,
errors: data.errors,
};
}
}
//# sourceMappingURL=annotation.js.map