comparative-judgement
Version:
Comparative Judgement Algorithms
211 lines (205 loc) • 6.65 kB
JavaScript
/* jshint -W024, expr:true */
/*jslint node: true */
/*global expect, fx, sinon*/
/*jshint -W083 */
;
var _ = require('underscore');
var csv = require('fast-csv');
var random = require('./random');
var selection = require('./selection');
var statutils = require('./statutils');
var btm = require('./btm');
var async = require('async');
var estimate = require('./estimation');
var simulate = function(nCands, nJudgements, algorithm, thr, AP, iters, seed, offset, rnd, meanTheta,sdTheta,callback){
var theta;
var players;
var results=[];
var alphaResults=[];
var iter = 0;
var decisions;
var decision;
var lastDec;
if(rnd) rnd=true;
async.series([
function(callback){
// Give me some candidates
if(!seed) seed = 1234;
if(!offset) offset = 0;
if(!meanTheta) meanTheta = 0;
if(!sdTheta) sdTheta=2;
theta = chooseCandidates(nCands,meanTheta,sdTheta,seed);
// Use the same players, and iterate through simulation
var iterations = _.range(parseInt(iters));
async.each(iterations, function(iter, callback) {
console.log('Processing iteration ' + iter);
// New set of decisions
decisions = [];
// Empty set of players
players=[];
for(var j=0; j<nCands; j++){
players.push(
{
_id:j,
selected:0,
comparisons:0,
observedScore:0,
trueScore:0,
seTrueScore:0,
opponents:[],
decisions:[],
theta:theta[j],
chosen:0,
lastTrueScore:0
}
);
}
var js = _.range(parseInt(nJudgements));
async.each(js, function(jmnt, callback) {
console.log('Processing judgement: ',jmnt+1);
decision = doJudgement(algorithm, players, thr, AP, offset, rnd);
decisions.push(decision);
//Fit the bt model
lastDec = decisions[decisions.length-1];
btm.btmModel(decisions, null, null, null, null, function(err, teams){
for(var k=0;k<players.length;k++){
var pl = _.find(teams, function(team) {return players[k]._id==team.team;});
if(pl){
players[k].chosen =0;
if(players[k]._id == lastDec.chosen | players[k]._id == lastDec.notChosen){
players[k].chosen=1;
}
players[k].lastTrueScore = players[k].trueScore;
players[k].trueScore = pl.theta;
players[k].seTrueScore = pl.seTheta;
}
players[k].iteration = iter + 1;
players[k].judgement = jmnt + 1;
var clonedPlayer = JSON.parse(JSON.stringify(players[k]));
results.push(clonedPlayer);
}
var alpha = estimate.estimateReliability(players);
alpha.iteration = iter + 1;
alpha.judgement = jmnt + 1;
alphaResults.push(alpha);
callback();
});
}, function(err){
if( err ) {
console.log('A judgement failed to process');
} else {
//console.log(decisions);
console.log('All judgements have been processed successfully');
callback();
}
});
}, function(err){
if( err ) {
console.log('An iteration failed to process: ',err);
} else {
console.log('All iterations have been processed successfully');
callback();
}
});
},
function(callback){
// do some more stuff ...
console.log('write out results now');
csv
.writeToPath("out.csv", results, {headers: true})
.on("finish", function(){
callback(null, 'results written to csv');
});
},
function(callback){
// do some more stuff ...
console.log('write out alpha results now');
csv
.writeToPath("alpha.csv", alphaResults, {headers: true})
.on("finish", function(){
callback(null, 'results written to csv');
});
}
],
// optional callback
function(err, msg){
callback(results.length);
});
};
var doJudgement = function(algorithm, players, thr, AP, offset, rnd){
if(!offset) offset=0;
var select;
if (algorithm=='adaptive'){
select = selection.selectionAdaptive;
} else if(algorithm=='swiss') {
select = selection.selectionSwiss;
} else {
select = selection.selectionNonAdaptive;
}
var pair;
// Get a pair
if(algorithm=='adaptive'){
pair = select(players, thr, AP, offset);
} else {
pair = select(players);
}
var left = _.find(players, function(player){ return player._id == pair[0]._id; });
var right = _.find(players, function(player){ return player._id == pair[1]._id; });
// Simulate a judgement
var j = simJudgement(left.theta, right.theta, rnd);
var k = j==1?0:1;
var decision;
var chosen = j==1?left._id:right._id;
var notChosen = j===0?left._id:right._id;
recordJudgement(left, j, right._id);
recordJudgement(right, k, left._id);
decision = {judge:'sim',chosen:chosen,notChosen:notChosen,timeTaken:0};
return decision;
};
var chooseCandidates = function(n, mean, sd, seed){
//return n candidates with a random normal distribution of theta
mean?mean:0;
sd?sd:2;
var Random = random.Random;
var stream1;
if(seed){
stream1 = new Random(seed);
} else {
stream1 = new Random();
}
// Create an array for simulation
var simCands = [];
for (var i=0; i<n; i++){
simCands.push(stream1.normal(mean,sd));
}
return simCands;
};
var simJudgement = function(theta, delta, rnd){
var Random = random.Random;
var rasch = statutils.rasch;
var p;
if(rnd){
p = rasch(0, 0);
} else {
p = rasch(theta, delta);
}
var stream1 = new Random();
var runif = stream1.random();
var judgement = runif < p?1:0;
return judgement;
};
var recordJudgement = function(player, win, opponent){
player.comparisons ++;
player.selected++;
player.observedScore += win;
if(!_.contains(player.opponents,opponent)){
player.opponents.push(opponent);
}
player.decisions.push(opponent);
};
module.exports = {
simulate:simulate,
chooseCandidates: chooseCandidates,
simJudgement: simJudgement,
recordJudgement: recordJudgement,
};