comparative-judgement/lib/interRater.js

Comparative Judgement Algorithms

// Read in a set of decisions
// Find all the judges in those decisions
// Select a random half of the judges
// Estimate the player stats
// Store as an iteration
// Save out to csv

/* jshint -W024, expr:true */
/*jslint node: true */
/*global expect, fx, sinon*/
/*jshint -W083 */

"use strict";
var _ = require('underscore');
var random = require('./random');
var selection = require('./selection');
var statutils = require('./statutils');
var estimation = require('./estimation');
var async = require('async');
var btm = require('./btm');
var pearson = require('./pearson');
var math= require('mathjs');

var interRaterReliability = function(iters, decisions, exclude, callback){
    var judges = [];
    var players = [];
    var players1 =[];
    var players2 =[];
    var myDecisions1;
    var myDecisions2;
    var corrs=[];
    var medianIr={median:0, sd:0, range:0, values:[]};
    // Check there are enough judges
    if(exclude){
      var bf = decisions.length;
      for (var i=0; i<exclude.length; i++){
        decisions = _.reject(decisions, function(decision){ return decision.judge == exclude[i];});
      }
      var aft = decisions.length;
      console.log(bf - aft, ' decisions removed');
    }
    judges = _.uniq(_.pluck(decisions, 'judge'));
    if (judges.length < 2) {
      return(callback(new Error('Not enough judges'),medianIr));
    } else {
      var n = Math.floor(judges.length / 2);
      var iterations = _.range(parseInt(iters));
      async.eachSeries(iterations, function(iter, callback) {
        // Perform operation here.
        console.log('Processing iteration ' + iter);
        var chosen  = _.uniq(_.pluck(decisions, 'chosen'));
        var notChosen = _.uniq(_.pluck(decisions, 'notChosen'));
        var playerIds = _.union(chosen, notChosen);
        var judgesChosen = _.sample(judges, n);
        var judgesNotChosen = _.difference(judges, judgesChosen);
        myDecisions1 = _.filter(decisions, function(decision){return _.contains(judgesChosen,decision.judge);});
        myDecisions2 = _.filter(decisions, function(decision){return _.contains(judgesNotChosen,decision.judge);});
        async.series([
          function(callback){
            players1 = [];
            for(var j =0; j<playerIds.length; j++){
              players1.push({_id: playerIds[j]});
            }
            btm.btmModel(myDecisions1, null, null, null, null, function(err, estPlayers){
              console.log('players 1 has returned');
              for(var k=0; k<players1.length; k++){
                var pl = _.find(estPlayers, function(p){return p.team == players1[k]._id;});
                if(pl) {
                  players1[k].theta = pl.theta;
                }
              }
              players1 = _.filter(players1, function(p){ return p.hasOwnProperty("theta");});
              callback();
            });
          },
          function(callback){
            players2 = [];
            for(var j =0; j<playerIds.length; j++){
              players2.push({_id: playerIds[j]});
            }
            btm.btmModel(myDecisions2, null, null, null, null, function(err, estPlayers){
              console.log('players2 has returned');
              for(var k=0; k<players2.length; k++){
                var pl = _.find(estPlayers, function(p){return p.team == players2[k]._id;});
                if(pl) {
                  players2[k].theta = pl.theta;
                }
              }
              players2 = _.filter(players2, function(p){ return p.hasOwnProperty("theta"); });
              callback();
            });
          },
          function(callback){
            console.log(players1.length, players2.length);
            //Now find union of two collections
            var coll1 = _.uniq(_.pluck(players1, '_id'));
            var coll2 = _.uniq(_.pluck(players2, '_id'));
            var playersToCor = _.intersection(coll1, coll2);
            console.log('correlating: ',playersToCor.length);
            //Remove all players not in the intersection
            players1 = _.filter(players1, function(pl){return playersToCor.indexOf(pl._id)>-1;});
            players2 = _.filter(players2, function(pl){return playersToCor.indexOf(pl._id)>-1;});
            //Only correlate where two scores exist
            players1 = _.sortBy(players1, '_id');
            players2 = _.sortBy(players2, '_id');
            var theta1 = _.pluck(players1, 'theta');
            var theta2 = _.pluck(players2, 'theta');
            var cor = pearson.correlation(theta1, theta2);
            corrs.push(cor);
            console.log('correlation: ',cor);
            callback();
          }],function(err, results){
            console.log('finished iteration', iter);
            callback();
          });
        }, function(err){
          var max = Math.max.apply(null, corrs);
          var min = Math.min.apply(null, corrs);
          var range = max - min;
          medianIr = {median:math.median(corrs), sd:math.std(corrs), range:range, values:corrs};
          console.log('All iterations have been processed successfully');
          callback(err, medianIr);
        });
    }
};

module.exports = {
  interRaterReliability:interRaterReliability,
};