grading/dist/cli/cmdPrepare.ts

Grading of student submissions, in particular programming tests.

import { OptionValues } from 'commander';
import fs from "fs";
import { createSubmitterDir, error, log, parseJsonWithComments, verb, verbosity, warn } from './cliUtil';
import { createClipSubmissionDir, rmClipSubmissionDir, runDocker, runPostCheckScript, runPreCheckScript } from './cmdCheck';
import { createLatexFragment } from "./cmdGrade";
import { prepareLatexFiles, runLatex } from "./cmdPDF";
import { createDir, ensureFileExists, ensureFolderExists, findDirContainingFile, folderExists, rmDir } from '../fsUtil';
import { doGrading, validate } from "../grade/grading";
import { GradingSchema } from "../grade/gradingschema";
import { Results } from "../grade/results";
import { generateGradingSchema } from "../grade/schemaGenerator";
import { Submitter } from "./submitter";
import { parse } from 'comment-json';
import path, { resolve } from 'path';

// .argument('[solutionFolder]', 'Folder with solution, should match structure of a submission', '.')
// .option('-r, --reportsDirSolution <reportsDir>',
//     'folder for generated JUnit and coverage reports, mounted in docker as /reports', './gradingOut/solution/reports')
// .addOption(optCheckDir)
// .addOption(optNpmCacheDir)
// .addOption(optCacheDir)
// .option('-g --gradingSchemaFile <schemaFile>', 'JSON file with grading schema (*.grading.json)', './solution/exam.grading.json')
// .option('-c --createGrading', 'Instead of grading, create grading scheme based on checks run on solution. This option specifies the file name. Cannot be used with -g', '')
// .option('-di, --dockerImage <dockerImage>', 'Docker image name', 'node:18.10-alpine')
// .option('-ds, --dockerScript <dockerScript>', 'Script to run in docker, this must be path in the vm', 'check/checkInDocker.sh')
// .option('-tod, --timeoutPerDockerRun <timeoutPerDockerRun>', 'Timeout per docker run (for each submission)', '300s')
// .option('-o --pdfDir <pdfDir>', 'The folder into which the PDF report is generated (using Moodle\'s folder structure).',
//     './gradingOut/solution/solutionReport')
// .option('--latexMainFile <latexMainFile>', 'Name of LaTeX main file (importing fragment via input)', "check/latex/grading.tex")
// .option('--latexFragmentFile <latexFragmentFile>', 'Name of LaTeX fragment (previously created via grade)', './gradingOut/solution/results.tex')
// .option('--workDir <workDir>', 'Name of working folder, this folder is cleaned before and after)', "./gradingOut/_working")
// .option('-l --latex <latex>', 'Name of LaTeX tool', 'xelatex')
// .option('--latexConsole', 'Show LaTex console output', false)
// .option('--validateOnly', 'Do not run tests, only validate grading schema')
// .addOption(optQuiet)
// .addOption(optVerbose)

export async function cmdPrepare(solutionFolder: string, options: OptionValues) {
    verbosity(options);
    log("Preparing grading.");
    try {

        const lecturer: Submitter = { name: 'Lecturer', submissionId: 'solution' };


        // part 1: run check on solution
        const checkDir = options.checkDir;
        const reportsDir = options.reportsDirSolution;
        const npmCacheDir = options.npmCacheDir;
        const cacheDir = options.cacheDir;
        const clipDir = options.clipDir;
        const schemaFile = options.gradingSchemaFile;
        const pdfDir = options.pdfDirSolution;
        const workDir = options.workDir;
        const validateOnly = options.validateOnly;
        const volumes = options.volumes;
        const patchFolder: string = options.patchFolder;

        const ignoredDirs = [...options.ignoredDirs, options.submissionsDir];

        if (!options.dockerUserDir) {
            program.error("dockerUserDir must be set.");
        }

        if (!validateOnly) {
            await ensureFolderExists(solutionFolder, "Check solution folder.");
            await ensureFolderExists(checkDir, "Check check folder.");
            if (npmCacheDir && ! await folderExists(npmCacheDir, 'npmCacheDir')) await createDir(npmCacheDir);
            if (cacheDir && ! await folderExists(cacheDir, 'cacheDir')) await createDir(cacheDir);
            if (clipDir != "" && ! await folderExists(clipDir, 'cacheDir')) await createDir(clipDir);
        }
        if (!validateOnly && !options.skipDocker) {
            if (await folderExists(reportsDir, 'reportsDirSolution')) await rmDir(reportsDir);
            await createDir(reportsDir);
        }

        const createGradingFile = (options.createGradingFile && options.createGradingFile.length > 0) ? options.createGradingFile : null;

        if (!createGradingFile) {
            await ensureFileExists(schemaFile, "Check grading schema file.");
        } else {
            if (validateOnly) {
                program.error("Validating a newly created grading file does not make any sense.");
            }
        }

        let projectDir = ".";
        try {
            projectDir = await findDirContainingFile(solutionFolder, options.projectFile, ignoredDirs);
            if (!validateOnly && !options.skipDocker) {
                verb("Project dir: " + projectDir);
            } else {
                log("Project dir: " + projectDir);
            }
        } catch (err) {
            warn(`Project file ${options.projectFile} not found in solution folder "${solutionFolder}", use "${projectDir}" (i.e. ${resolve(projectDir)}).`);
        }

        if (!validateOnly && !options.skipDocker) {
            const clipUserDir = await createClipSubmissionDir(clipDir, lecturer.submissionId);
            await runPreCheckScript(options.preCheckScript, projectDir, clipUserDir, checkDir, true, options);
            try {
                await runDocker("prepare", false, checkDir, projectDir, reportsDir, npmCacheDir, cacheDir, clipDir, volumes, lecturer.submissionId,
                    {
                        dry: options.dry,
                        dockerImage: options.dockerImage,
                        dockerUserDir: options.dockerUserDir,
                        dockerWorkDir: options.dockerWorkDir,
                        dockerShellCmd: options.dockerShellCmd,
                        dockerScript: options.dockerScript,
                        dockerArgs: options.dockerArgs,
                        timeoutPerDockerRun: options.timeoutPerDockerRun,
                        preserveContainer: options.preserveContainer
                    },
                    options
                );
            } finally {
                await runPostCheckScript(options.postCheckScript, projectDir, clipUserDir, checkDir, true, options);
                if (clipUserDir) {
                    await rmClipSubmissionDir(clipUserDir, lecturer.submissionId);
                }
            }

        } else if (options.skipDocker) {
            log("Skipping Docker run.");
        }

        // part 2a: create grading scheme
        if (createGradingFile) {
            log(`Generating grading schema from check results (submission id ${lecturer.submissionId}).`)
            const gradingSchema = await generateGradingSchema(reportsDir, lecturer);
            await fs.promises.writeFile(createGradingFile, JSON.stringify(gradingSchema, null, 4), { encoding: "utf-8" });
            log(`Initial grading schema written to ${createGradingFile}`);
        } else {
            // part 2b: grade solution
            const gradingSchema = await parseJsonWithComments(schemaFile);
            if (validateOnly) {
                if (!validate(gradingSchema)) {
                    program.error("There were validation errors.");
                } else {
                    log(`${SEP}\nDone.`);
                    return; // ok
                }
            }

            let results: Results = await doGrading({
                gradingSchema, manualCorrection: undefined, submitters: [lecturer], reportsDir,
                startMarker: options.testOutputStartMarker, endMarker: options.testOutputEndMarker, patchFolder
            });


            if (!options.latexFragmentFileSolution) {
                program.error("Missing option --latexFragmentFileSolution");
            }


            await createLatexFragment(results, { latexFragmentFile: options.latexFragmentFileSolution, dry: options.dry, workDir: workDir, reportsDir: reportsDir, patchFolder: patchFolder });

            // part 3: create PDF
            if (!options.skipPDF) {
                const latex = options.latex;
                const latexMainFile = options.latexMainFile;
                await ensureFileExists(latexMainFile, "Check option --latexMainFile.");
                const latexFragmentFile = options.latexFragmentFileSolution;
                const latexMainFolderCopy = options.latexMainFolderCopy;
                await ensureFileExists(latexFragmentFile, "Check option --latexFragmentFileSolution.");
                if (await folderExists(pdfDir, 'pdfDirSolution')) await rmDir(pdfDir);
                if (await folderExists(workDir, 'workDir')) await rmDir(workDir);
                await createDir(pdfDir)
                await createDir(workDir)
                const { latexMainLatex, latexMainPDF, latexMainWOExt } = prepareLatexFiles(latexMainFile, workDir, latexFragmentFile, latexMainFolderCopy);
                await runLatex(createSubmitterDir(lecturer), latexMainLatex, pdfDir, latex, workDir, latexMainPDF, latexMainWOExt, options.latexConsole, options);
            } else {
                log("Skipping PDF generation.");
            }
        }
    } catch (err) {
        error(`${SEP}`)
        if (err instanceof Error) {
            verb(`${err.stack}`);
        }
        // error(`${SEP}\nError: ${err}`);
        program.error(String(err));
    }
    log(`${SEP}\nDone.`);

}