@vfarcic/dot-ai
Version:
AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance
109 lines (108 loc) • 4 kB
JavaScript
;
/**
* Dataset Loader for Standard OpenAI Evals Format
*
* Loads JSONL evaluation datasets following OpenAI Evals standard:
* - Each line contains: {input, ideal, metadata}
* - Supports filtering by category, complexity, tags
* - Used by both integration tests and evaluation framework
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.loadEvalDataset = loadEvalDataset;
exports.loadTestPhase = loadTestPhase;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
/**
* Load evaluation dataset from JSONL file
* @param datasetName - Name of the dataset file (without .jsonl extension)
* @param filter - Optional filter criteria
* @returns Array of evaluation samples
*/
function loadEvalDataset(datasetName, filter) {
const datasetsDir = path.join(process.cwd(), 'eval', 'datasets');
const datasetPath = path.join(datasetsDir, `${datasetName}.jsonl`);
if (!fs.existsSync(datasetPath)) {
throw new Error(`Dataset not found: ${datasetPath}`);
}
const fileContent = fs.readFileSync(datasetPath, 'utf8');
const lines = fileContent
.trim()
.split('\n')
.filter(line => line.trim());
const samples = lines.map((line, index) => {
try {
return JSON.parse(line);
}
catch (error) {
throw new Error(`Invalid JSON at line ${index + 1} in ${datasetName}.jsonl: ${error}`, { cause: error });
}
});
// Apply filters if provided
if (filter) {
return samples.filter(sample => {
if (filter.category && sample.metadata.category !== filter.category) {
return false;
}
if (filter.complexity &&
sample.metadata.complexity !== filter.complexity) {
return false;
}
if (filter.phase && sample.metadata.phase !== filter.phase) {
return false;
}
if (filter.tool && sample.metadata.tool !== filter.tool) {
return false;
}
if (filter.tags) {
const hasAllTags = filter.tags.every(tag => sample.metadata.tags.includes(tag));
if (!hasAllTags) {
return false;
}
}
return true;
});
}
return samples;
}
/**
* Load samples for a specific test phase
* @param datasetName - Dataset name
* @param phase - Test phase to load
* @returns Array of samples for that phase
*/
function loadTestPhase(datasetName, phase) {
return loadEvalDataset(datasetName, { phase });
}