grading/dist/cli/cmdGrade.ts

Grading of student submissions, in particular programming tests.

import { OptionValues } from "commander";
import fs from "fs";
import path from "path";
import { pipeline } from "stream/promises";
import { CSVProducer, readCSV, writeTableWithEncoding } from "../csv";
import { ensureFileExists, ensureFolderExists, fileExists, readDir, timestamp } from "../fsUtil";
import { doGrading } from "../grade/grading";
import { GradingSchema, readGradingSchema } from "../grade/gradingschema";
import { GRADE_VALUES, Results } from "../grade/results";
import { toTable } from "../grade/results2csv";
import { toLatex } from "../grade/results2latex";
import { computeStatistics } from "../grade/statistics";
import { error, generateFileName, JSONSrcMap, log, parseJsonWithComments, parseSubmitter, quickFormat, retrieveOriginalMoodleFile, verb, verbosity, withDecimals } from "./cliUtil";

import { warn } from "console";
import { CorrectionSchema } from "../grade/correctionschema";
import { Table } from "../table";
import { compareResults } from "./cmdCompare";
import { Submitter } from "./submitter";

const ENCODING: BufferEncoding = "utf-8";


export async function cmdGrade(moodleFile: string, options: OptionValues, runInternally = false) {
    verbosity(options);

    log("Grade all results at " + timestamp());

    try {
        moodleFile = await retrieveOriginalMoodleFile(moodleFile, options);

        const reportsDir: string = options.reportsDir;
        const submissionsDir: string = options.submissionsDir;
        const schemaFile = options.gradingSchemaFile;

        await ensureFolderExists(submissionsDir, "Check submission folder.");
        await ensureFolderExists(reportsDir, "Check reports folder.");
        await ensureFileExists(schemaFile, "Check grading schema file.");

        const patchFolder: string = options.patchFolder;

        if (moodleFile.length == 0) {
            moodleFile == null;
        }

        if (!options.noMoodleCSV) {
            if (!moodleFile) {
                program.error("No Moodle input CSV file provided, cannot create output grading file.")
            }
            await ensureFileExists(moodleFile, "Check MoodleCSV setting.");
        }

        const submissionDirs = await readDir(submissionsDir, 'dir', false, 'Check submissions folder.');
        let submitters = submissionDirs.map(subDir => parseSubmitter(subDir));
        if (submitters.length == 0) {
            program.error(`No submissions found in ${submissionsDir}`)
        }

        const gradingSchema = await readGradingSchema(schemaFile);
        const manualCorrection = await readManualCorrections(gradingSchema, options);
        checkManualCorrections(gradingSchema, submitters, manualCorrection, false);
        let results: Results = await doGrading({
            gradingSchema, manualCorrection, submitters,
            reportsDir, startMarker: options.testOutputStartMarker, endMarker: options.testOutputEndMarker, patchFolder
        });


        if (!options.noLatex) {
            const latexFragmentFile = options.latexFragmentFile;
            try {
                if (!options.latexFragmentFile) {
                    program.error("Missing option --latexFragmentFile");
                }
                const workToReportRelPath = path.relative(options.workDir, reportsDir);
                await createLatexFragment(results,
                    { latexFragmentFile: latexFragmentFile, dry: options.dry, workDir: options.workDir, reportsDir: reportsDir, patchFolder: patchFolder });
            } catch (err) {
                error("Error creating LaTex fragment: " + err);
            }
        }
        let resultFile = undefined;
        if (!options.noResultCSV) {
            try {
                resultFile = await createResultCSV(gradingSchema, results, options);
            } catch (err) {
                error("Error creating result CSV: " + err);
            }
        }
        let moodleGradingTable: Table | null = null;
        if (!options.noMoodleCSV) {

            try {
                const readable = fs.createReadStream(moodleFile, "utf-8");
                moodleGradingTable = await readCSV(readable);
            } catch (err) {
                error(`Error reading Moodle grading CSV '${moodleFile}'`);
                error(err);
            }

            if (moodleGradingTable) {
                await createGradingFile(moodleGradingTable, gradingSchema, results, options, moodleFile, []);
            }

        }

        await statistics(submitters, gradingSchema, results, moodleGradingTable, options);

        verb(`Compare results with previous results: ${options.compareResults}`);
        if (options.compareResults && resultFile) {
            await compareResults(resultFile, options);
        }

    } catch (err) {
        error(`${SEP}\nError: ${err}`);
        program.error(String(err));
    }
    if (!runInternally) {
        log(`${SEP}\nDone.`);
    }

}


async function statistics(submitters: Submitter[], gradingSchema: GradingSchema, results: Results, moodleGradingTable: Table | null, options: OptionValues) {

    const sumCheckReports = results.studentResults.reduce((sum, studentResult) => sum + (studentResult.checkReportFound ? 1 : 0), 0)
    const sumStudentReports = results.studentResults.reduce((sum, studentResult) => sum + (studentResult.studentReportFound ? 1 : 0), 0)
    const sumGradingDone = results.studentResults.reduce((sum, studentResult) => sum + (studentResult.gradingDone ? 1 : 0), 0)

    const grades: Map<string, number> = new Map();
    for (let i = 0; i <= 15; i++) { grades.set(GRADE_VALUES[i], 0); }
    results.studentResults.forEach(studentResult => {
        if (!studentResult.noSubmission) {
            let count = grades.get(studentResult.grade);
            if (count !== undefined) {
                count++;
                grades.set(studentResult.grade, count);
            }
        }
    });

    const TAB = 25;
    const table = new Table();
    if (moodleGradingTable) {
        const participants = moodleGradingTable.rowsCount - 1;
        logStatTab(table, TAB, "Participants", withDecimals(participants));
        if (participants > 0) {
            logStatTab(table, TAB, "Submission rate", withDecimals(submitters.length / participants));
        } else {
            table.addRow("Submission rate", "na");
        }
    } else {
        table.addRow("Participants", "na");
        table.addRow("Submission rate", "na");
    }

    logStatTab(table, TAB, "Submissions", withDecimals(submitters.length));
    logStatTab(table, TAB, "Missing check test reports", withDecimals(submitters.length - sumCheckReports));
    logStatTab(table, TAB, "Missing student test reports", withDecimals(submitters.length - sumStudentReports));
    logStatTab(table, TAB, "Not graded successfully", withDecimals(submitters.length - sumGradingDone));


    let gradeSum = 0;
    let gradeCount = 0;
    for (let i = 0; i < GRADE_VALUES.length; i++) {
        const count = grades.get(GRADE_VALUES[i]) ?? -1;
        gradeSum += count * Number.parseFloat(GRADE_VALUES[i].replace(',', '.'));
        gradeCount += count;
    }
    if (gradeCount > 0) {
        logStatTab(table, TAB, "Average grade", withDecimals(gradeSum / gradeCount));
        logStatTab(table, TAB, "Failure rate", withDecimals((grades.get(GRADE_VALUES[0]) ?? -1) / gradeCount));
    } else {
        table.addRow("Average grade", "na");
        table.addRow("Failure rate", "na");
    }

    logStatTab(table, 5, "Grade", ...GRADE_VALUES);
    logStatTab(table, 5, "from", ...results.gradingTable);
    logStatTab(table, 5, "count", ...GRADE_VALUES.map(gv => grades.get(gv)));

    const statistics = computeStatistics(gradingSchema, results);
    logStatTab(table, TAB, "Average coverage", withDecimals(statistics.averageCoverage));
    logStatTab(table, TAB, "Average student tests", withDecimals(statistics.averageStudentTests));
    logStatTab(table, TAB, "Applied corrections", statistics.appliedCorrections);
    logStatTab(table, 20, "Label", "Success Rate")
    for (const item of statistics) {
        if (item.itemType === 'exam' || item.itemType === 'task') {
            logStatTab(table, 20, item.title, withDecimals(item.successRate));
        } else {
            verbStatTab(table, 20, item.title, withDecimals(item.successRate));
        }
    }

    if (!options.noStatisticsCSV) {
        const fileName = generateFileName(options.statisticsFile, { examName: results.examName, dateTime: results.dateTime });
        if (options.dry) {
            log(`Would write statistics "${fileName}", skipped in dry mode.`);
            return;
        } else {
            log(`Write statistics "${fileName}".`);

        }
        await writeTableWithEncoding(fileName, table, options.encoding, options.resultCSVDelimiter);
    }
}

export async function createLatexFragment(results: Results,
    optionsLatexFragment: {
        latexFragmentFile: string, dry: boolean,
        workDir: string, reportsDir: string,
        patchFolder: string
    }) {
    const latexString = toLatex(results, optionsLatexFragment.workDir, optionsLatexFragment.reportsDir, optionsLatexFragment.patchFolder);

    if (optionsLatexFragment.dry) {
        log(`Would write latex fragment "${optionsLatexFragment.latexFragmentFile}", skipped in dry mode.`);
        return;
    }
    log(`Write latex fragment "${optionsLatexFragment.latexFragmentFile}".`);
    await fs.promises.writeFile(optionsLatexFragment.latexFragmentFile, latexString);
}

async function createResultCSV(gradingSchema: GradingSchema, results: Results, options: OptionValues) {
    const fileName = generateFileName(options.resultFile, { examName: results.examName, dateTime: results.dateTime });

    const table = toTable(results, gradingSchema);
    if (options.dry) {
        log(`Would write result CSV file "${fileName}", skipped in dry mode.`);
        return;
    }
    log(`Write result CSV "${fileName}".`);
    await writeTableWithEncoding(fileName, table, options.encoding, options.resultCSVDelimiter);
    return fileName;
}

/**
 * Writes Moodle Grading File (Bewertungstabelle...)
 */
async function createGradingFile(moodleGradingTable: Table, gradingSchema: GradingSchema, results: Results, options: OptionValues, moodleFile: string, selected: string[]) {
    const moodleFileBase = path.basename(moodleFile);
    const fileName = generateFileName(options.gradedMoodleFile,
        {
            examName: results.examName,
            dateTime: results.dateTime,
            moodleFile: moodleFileBase.substring(0, moodleFileBase.length - path.extname(moodleFileBase).length),
            selected: selected
        },
        true);

    // we need to work on original table, otherwise Moodle does not accept changes
    // const outTable = new Table(moodleGradingTable); 

    if (options.gradingValue !== "grade" && options.gradingValue !== "points") {
        throw new Error(`Invalid grading value "${options.gradingValue}", must be either grade or points.`);
    }

    const useGrade = options.gradingValue === 'grade';
    verb(`Use ${useGrade ? 'grade' : 'points'} for grading in Moodle result table.`)

    results.studentResults.forEach(studentResult => {

        if (!studentResult.noSubmission && (selected?.length === 0 || selected.includes(studentResult.submissionId))) {
            const row = moodleGradingTable.findRowWhere(row => {
                const longID = moodleGradingTable.at("ID", row);
                if (longID === "Teilnehmer/in" + studentResult.submissionId) { // first column in Moodle file
                    return true;
                }
                return false;
            });
            if (row < 1) {
                warn(`Submission ${studentResult.submissionId} (${studentResult.userName}) not found.`);
            } else {
                //  outTable.addRowFromTable(moodleGradingTable, row); // copy row
                if (useGrade) {
                    moodleGradingTable.setAt("Bewertung", row, gradeToNumber(studentResult.grade));

                } else { // use points 
                    moodleGradingTable.setAt("Bewertung", row,
                        new Intl.NumberFormat('de-DE', { minimumFractionDigits: 2 }).format(studentResult.points));
                }
                const date = new Date();
                const dateString = date.toLocaleDateString('de-DE', { weekday: 'long', year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: 'numeric' });

                moodleGradingTable.setAt("Zuletzt geändert (Bewertung)", row, dateString);
            }
        }

    });

    if (options.dry) {
        log(`Would write Moodle grading file "${fileName}", skipped in dry mode.`);
        return;
    }
    log(`Write Moodle grading file "${fileName}".`);

    const writeStreamBew = fs.createWriteStream(fileName, { encoding: ENCODING });
    const csvStreamBew = new CSVProducer(moodleGradingTable, { delimiter: ",", escapeWhenSpace: true, encoding: ENCODING });
    await pipeline(csvStreamBew, writeStreamBew);
}


function percent(successRate: number) {
    const n = Math.round(successRate * 1000)
    if (n % 10 == 0) {
        return (n / 10) + ".0%";
    }
    return (n / 10) + "%";
}

function logStatTab(table: Table, pad: number, ...values: any[]) {
    table.addRow(...values)
    let out = "";
    values.forEach((v, i) => {
        if (i != 0) {
            out += ": ";
        }
        out += quickFormat(v, pad);
    });
    log(out);
}

function verbStatTab(table: Table, pad: number, ...values: any[]) {
    table.addRow(...values)
    let out = "";
    values.forEach((v, i) => {
        if (i != 0) {
            out += ": ";
        }
        out += quickFormat(v, pad);
    });
    verb(out);
}
async function readManualCorrections(gradingSchema: GradingSchema, options: OptionValues) {
    if (!options.manualCorrectionsFile) {
        return;
    }
    const correctionFile = generateFileName(options.manualCorrectionsFile, { stdInFolder: options.stdInFolder });

    if (! await fileExists(correctionFile, 'correctionFile')) {
        verb(`No manual correction file "${correctionFile}" found, skipped.`);
        return undefined;
    }

    const correctionSchema: CorrectionSchema = await parseJsonWithComments(correctionFile);


    if (gradingSchema.exam !== correctionSchema.exam) {
        throw new Error(`Exam name "${gradingSchema.exam}" does not match correction exam name "${correctionSchema.exam}".`);
    }
    if (gradingSchema.course !== correctionSchema.course) {
        throw new Error(`Course name "${gradingSchema.course}" does not match correction course name "${correctionSchema.course}".`);
    }
    if (gradingSchema.term !== correctionSchema.term) {
        throw new Error(`Term name "${gradingSchema.term}" does not match correction term name "${correctionSchema.term}".`);
    }

    let deprecatedUserIDFound = false;
    correctionSchema.corrections.forEach(correction => {
        if (!correction.submissionID && "userID" in correction && typeof correction.userID === "string") { // fix old format
            correction.submissionID = correction.userID;
            deprecatedUserIDFound = true;
        }
        if (correction.tasks) {
            correction.tasks = correction.tasks.filter(task => task.points || task.reason);
        }
        if (correction.general) {
            correction.general = correction.general.filter(task => task.points || task.reason); // filter out empty lines
            correction.general.forEach(general => { if (!general.absolute) { general.absolute = false; } }); // explicitly set boolean value
            if (correction.general.filter(general => general.absolute).length > 1) {
                throw new Error(`Only one absolute correction allowed per student, found more for ${correction.submissionID} ${correction.userName}.`);
            }
        }
    });
    correctionSchema.corrections = correctionSchema.corrections.filter(correction =>
        (correction.tasks?.length && correction.tasks?.length > 0)
        ||
        (correction.general?.length && correction.general?.length > 0)
    );

    if (deprecatedUserIDFound) {
        warn(`Deprecated userID found in manual correction file "${correctionFile}", please use submissionID instead.`);
    }
    return correctionSchema;
}

function checkManualCorrections(gradingSchema: GradingSchema, submissions: Submitter[], manualCorrection: CorrectionSchema | undefined, selectionOnly: boolean) {
    if (!manualCorrection) {
        return;
    }
    manualCorrection.corrections.forEach(correction => {
        const submission = submissions.find(sub => sub.submissionId === correction.submissionID);
        if (!submission && !selectionOnly) {
            const possibleID = submissions.find(sub => sub.name === correction.userName)?.submissionId || undefined;
            throw new Error(`Cannot apply manual correction: No submission with ID ${correction.submissionID} (${correction.userName}) found in submissions${possibleID ? ". Did you mean " + possibleID : " Neither found student name in submissions."}.`);
        }
        if (submission) {
            if (!submissions.find(sub => sub.name === correction.userName)) {
                throw new Error(`Cannot apply manual correction: Submission with ID ${correction.submissionID} found, but names are different: Did you mean ${submission.name} instead of ${correction.userName}?`);
            }
            if (submission.name !== correction.userName) {
                const possibleID = submissions.find(sub => sub.name === correction.userName)?.submissionId || undefined;
                throw new Error(`Error in manual correction: Student name and id do not match. Did you mean ${submission.name}?`
                    + (possibleID ? ` Student ${correction.userName} has ID ${possibleID}.` : ""));
            }
            correction.tasks?.forEach(task => {
                const gradingTask = gradingSchema.tasks.find(t => t.name === task.name);
                if (!gradingTask) {
                    throw new Error(`Error in manual correction: No task with name ${task.name} found in grading schema.`);
                }
            });
        }
    });


}

function gradeToNumber(grade: string): number {
    const n = Number.parseFloat(grade.replace(',', '.'));
    if (Number.isNaN(n)) {
        throw new Error(`Invalid grade "${grade}"`);
    }
    return n;
}