grading/dist/cli/cmdCompare.ts

Grading of student submissions, in particular programming tests.

import { OptionValues } from "commander";
import fs from "fs";

import { error, log, quickFormat, verb, verbosity, withDecimals } from "./cliUtil";

import path from "path";
import { fileExists, readDir } from "../fsUtil";
import { loadTableWithEncoding, parseGrading } from "../csv";



interface Change {
    submissionId: string,
    name: string,
    oldGrade: number,
    oldPoints: number,
    newGrade: number,
    newPoints: number,
    delta: number
}

export async function compareResults(newCSV: string, options: OptionValues) {
    if (!await fileExists(newCSV)) {
        return;
    }
    const dir = path.dirname(newCSV);
    const newCSVRel = path.basename(newCSV);
    const foundPrefix = newCSVRel.match(/^[^0-9]+/);
    if (!foundPrefix) {
        verb("Cannot compare results automatically, no prefix (without digits) found in file name.");
        return;
    }
    const prefix = foundPrefix[0];
    verb(`Looking for old results with prefix '${prefix}' (and extension '.csv') in '${dir}'`);
    const oldResults = (await readDir(dir, "file", false)).filter(f => {
        // verb(`Found file '${f}'`);
        return f !== newCSVRel && f.startsWith(prefix) && f.endsWith(".csv")
    })
    if (oldResults.length === 0) {
        verb("Cannot compare results automatically, no old result found.");
        return;
    }
    const latestResult = oldResults.map(f => path.join(dir, f)).sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs)[0];

    cmdCompare(latestResult, newCSV, options, true);
}

export async function cmdCompare(oldCSV: string, newCSV: string, options: OptionValues, runInternally = false) {
    verbosity(options);

    if (oldCSV === newCSV) {
        program.error("Comparing the same file doesn't make any sense.");
    }
    log(`Compare '${oldCSV}' (old) with '${newCSV}' (new)`)
    try {

        const oldResult = await loadTableWithEncoding(oldCSV, options.encoding, options.resultCSVDelimiter);
        const oldColSubmissionID = oldResult.getColForTitles('submissionID', "userID");
        const oldColName = oldResult.getColForTitles('name');
        const oldColSubmissionFlag = oldResult.getColForTitles('abgabe', "submitted");
        const oldColGrade = oldResult.getColForTitles('note', "grade");
        const oldColPoints = oldResult.getColForTitles('punkte', "points");

        const newResult = await loadTableWithEncoding(newCSV, options.encoding, options.resultCSVDelimiter);
        const newColSubmissionID = newResult.getColForTitles('submissionID', "userID");
        const newColName = newResult.getColForTitles('name');
        const newColSubmissionFlag = newResult.getColForTitles('abgabe', "submitted");
        const newColGrade = newResult.getColForTitles('note', "grade");
        const newColPoints = newResult.getColForTitles('punkte', "points");

        const changes: Change[] = [];

        let oldSum = 0;
        let newSum = 0;
        let oldCount = 0;
        let newCount = 0;

        const oldOnly: string[] = [];
        const newOnly: string[] = [];

        for (let oldRow = 2; oldRow <= oldResult.rowsCount; oldRow++) {
            oldCount++;
            const submissionID = oldResult.getText(oldColSubmissionID, oldRow);
            const name = oldResult.getText(oldColName, oldRow);
            const oldGrade = parseGrading(oldResult.getText(oldColGrade, oldRow));
            const oldPoints = parseGrading(oldResult.getText(oldColPoints, oldRow));
            const newRow = newResult.findRowWhere(row => newResult.getText(newColSubmissionID, row) === submissionID);

            oldSum += oldGrade;

            let newGrade = 0;
            let newPoints = 0;
            if (newRow > 0) {
                newGrade = parseGrading(newResult.getText(newColGrade, newRow));
                newPoints = parseGrading(newResult.getText(newColPoints, newRow));
                newSum += newGrade;
                newCount++;
                changes.push({ submissionId: submissionID, name: name, oldGrade: oldGrade, oldPoints: oldPoints, newGrade: newGrade, newPoints: newPoints, delta: newGrade - oldGrade })
            } else {
                oldOnly.push(quickFormat(submissionID, 8) + ", " + quickFormat(name.substring(0, 20), 20) + ": " + quickFormat(withDecimals(oldGrade), 3));
            }


        }

        for (let newRow = 2; newRow <= newResult.rowsCount; newRow++) {
            const submissionID = newResult.getText(newColSubmissionID, newRow);
            const name = newResult.getText(newColName, newRow);
            const newGrade = parseGrading(newResult.getText(newColGrade, newRow));
            const newPoints = parseGrading(newResult.getText(newColPoints, newRow));
            const oldRow = oldResult.findRowWhere(row => oldResult.getText(oldColSubmissionID, row) === submissionID);

            if (oldRow <= 0) {
                newSum += newGrade;
                newCount++;
                newOnly.push(`${quickFormat(submissionID, 8)}, ${quickFormat(name.substring(0, 20), 20)}: ${quickFormat(withDecimals(newGrade), 3)} (${quickFormat(withDecimals(newPoints), 3)})`);
            }
        }

        const improved = changes.filter(c => c.delta < 0).length;
        const deteriorated = changes.filter(c => c.delta > 0).length;
        const unchanged = changes.filter(c => c.delta === 0).length;
        const avg1 = oldCount ? oldSum / oldCount : 0;
        const avg2 = newCount ? newSum / newCount : 0;

        if (!options.summaryOnly) {
            log("- Individual changes:");
            (options.showUnchanged ? changes : changes.filter(c => c.delta !== 0)).forEach(c => {
                log(`    - ${quickFormat(c.submissionId, 8)}, ${quickFormat(c.name.substring(0, 20), 20)}: ${quickFormat(withDecimals(c.oldGrade), 3)} (${quickFormat(withDecimals(c.oldPoints), 3)}) <-> ${quickFormat(withDecimals(c.newGrade), 3)} (${quickFormat(withDecimals(c.newPoints), 3)}) = ${withDecimals(c.delta)}`);
            })

            if (oldOnly.length || newOnly.length) {
                if (oldOnly.length) {
                    log("- Submitters only in 1st (old) result: " + oldOnly.length);
                    oldOnly.forEach(s => log("    - " + s));
                }
                if (newOnly.length) {
                    log("- Submitters only in 2nd (new) result: " + newOnly.length);
                    newOnly.forEach(s => log("    - " + s));
                }

            }
        }
        log("Summary:")
        log("- " + quickFormat("Improved: ", 10) + improved);
        log("- " + quickFormat("Deteriorated: ", 10) + deteriorated);
        log("- " + quickFormat("Unchanged: ", 10) + unchanged);
        log("- " + quickFormat("Submissions: ", 10) + oldCount + " <--> " + newCount);
        log("- " + quickFormat("Averages: ", 10) + withDecimals(avg1) + " <--> " + withDecimals(avg2));
        if (options.summaryOnly) {
            if (oldOnly.length) {
                log("- Submitters only in 1st (old) result: " + oldOnly.length);
            }
            if (newOnly.length) {
                log("- Submitters only in 2nd (new) result: " + newOnly.length);
            }
        }

    } catch (err) {
        error(`${SEP}\nError: ${err}`);
        program.error(String(err));
    }
    if (!runInternally) {
        log(`${SEP}\nDone.`);
    }

}