grading/dist/cli/cmdCheck.ts

Grading of student submissions, in particular programming tests.

import { OptionValues } from 'commander';
import * as fs from 'fs';
import path from 'path';
import { hrtime } from 'process';
import { absPath, createDir, ensureFileExists, ensureFolderExists, execShell, fileExists, findDirContainingFile, folderExists, isEmptyDir, readDir, rmDir, rmFile, timestamp, unzip } from '../fsUtil';
import { getPatchFileName, gitAddAndCommit, gitApplyPatch, gitCreateAndCheckoutBranch, hasLocalGitRepository } from '../gitCommands';
import { error, generateFileName, isSubmissionSelected, log, parseSubmitter, retrieveSelectedFilter, retrieveZipFile, timeDiff, verb, verbosity, warn } from './cliUtil';
import { Submitter } from './submitter';

export async function cmdCheckAction(zipFile: string, options: OptionValues) {
    return await cmdCheck(zipFile, options);
}

function appendLog(logfile: string, message?: any, ...optionalParams: any[]) {
    let out = message ?? "";
    if (optionalParams) {
        out += " " + optionalParams.join(" ");
    }
    out += "\n";
    fs.appendFileSync(logfile, out, { encoding: "utf-8" });
}

async function rewireConsoleOutput(logOutput: string) {
    if (!logOutput) {
        return;
    }

    const logFile = generateFileName(logOutput, {
        dateTime: Date.now()
    });
    if (await fileExists(logFile)) {
        log(`Reset log file ${logFile}`);
        rmFile(logFile);
    }
    const parentFolder = path.dirname(logFile);
    if (! await fileExists(parentFolder)) {
        await createDir(parentFolder);
    }

    const orgConsoleLog = console.log;
    const orgConsoleError = console.error;
    const orgConsoleWarn = console.warn;

    console.log = function (message?: any, ...optionalParams: any[]) {
        orgConsoleLog(message, ...optionalParams);
        appendLog(logFile, message, ...optionalParams);
    }
    console.error = function (message?: any, ...optionalParams: any[]) {
        orgConsoleError(message, ...optionalParams);
        appendLog(logFile, message, ...optionalParams);
    }
    console.warn = function (message?: any, ...optionalParams: any[]) {
        orgConsoleWarn(message, ...optionalParams);
        appendLog(logFile, message, ...optionalParams);
    }

    orgConsoleLog(`Write all output to ${logFile}`)
}

export async function cmdCheck(zipFile: string, options: OptionValues, runInternally = false, errors?: string[]) {
    verbosity(options);
    if (!errors) {
        errors = [];
    }

    const orgConsoleLog = console.log;
    const orgConsoleError = console.error;
    const orgConsoleWarn = console.warn;

    try {
        const logOutput = options.logOutput;
        await rewireConsoleOutput(logOutput);


        const startTime = hrtime.bigint();

        if (runInternally) {
            log("Start running tests on submissions.");
        } else {
            log("Start running tests on submissions at " + timestamp());
        }

        try {
            zipFile = await retrieveZipFile(zipFile, options);
            await ensureFileExists(zipFile, "Check zip file.");

            const submissionsDir = options.submissionsDir;
            const checkDir = options.checkDir;
            const clipDir = options.clipDir;
            const reportsDir = options.reportsDir;
            const npmCacheDir = options.npmCacheDir;
            const cacheDir = options.cacheDir;
            const dry = options.dry;
            const prepareSubmissionCmd = parsePrepareSubmissionCmd(options.prepareSubmissionCmd);
            const onlyPrepareSubmission = options.onlyPrepareSubmission;
            const skipPrepareSubmission = options.skipPrepareSubmission;
            const volumes = options.volumes;
            const skipStudentTests = options.skipStudentTests;

            const patch: boolean = options.patch;
            const patchFolder: string = options.patchFolder;
            const patchGradingBranch: string = options.patchGradingBranch;

            if (onlyPrepareSubmission && skipPrepareSubmission) {
                program.error(`--skipPrepareSubmission and --onlyPrepareSubmission are mutually exclusive.`);
            }
            if (onlyPrepareSubmission && !prepareSubmissionCmd) {
                warn(`--onlyPrepareSubmission is set, but no prepareSubmissionCmd is defined. Nothing to do.`);
            }

            const selected = retrieveSelectedFilter(options);
            const selectPatched = options.selectPatched;
            const patchedSubmissions: Submitter[] = [];
            const cleanBefore = options.cleanBefore;
            if (!cleanBefore) {
                verb('Do not clean before.');
            }
            const max = Number.parseInt(options.max);
            const fromSubmission = Number.parseInt(options.fromSubmission);
            const toSubmission = Number.parseInt(options.toSubmission);

            await ensureFolderExists(checkDir, "Check check folder.");

            if (npmCacheDir && ! await folderExists(npmCacheDir, 'npmCacheDir')) await createDir(npmCacheDir);
            if (cacheDir && ! await folderExists(cacheDir, 'cacheDir')) await createDir(cacheDir);
            if (clipDir && ! await folderExists(clipDir, 'clipDir')) await createDir(clipDir);
            if (cleanBefore && await folderExists(submissionsDir, 'submissionsDir')) {
                verb(`Remove submissions folder ${submissionsDir} (clean).`);
                await rmDir(submissionsDir);
            }
            if (! await folderExists(submissionsDir, 'folderExists')) await createDir(submissionsDir);
            if (cleanBefore && !dry && !skipStudentTests && await folderExists(reportsDir, 'reportsDir')) await rmDir(reportsDir); // don't clear on dry run
            if (! await folderExists(reportsDir, 'reportsDir')) await createDir(reportsDir);

            await unzip(zipFile, submissionsDir); // we always unzip the big file

            const submissionDirs = await readDir(submissionsDir, 'dir', false, 'Check submissions folder.');
            if (submissionDirs.length == 0) {
                program.error(`No submissions found in ${submissionsDir}`)
            }
            let name = "", subId = ""; // define here for catch block
            // toSubmission and fromSubmission are 1-based
            // toSubmissionChecked is exclusive
            let toSubmissionChecked = toSubmission > 0 ? Math.min(toSubmission, submissionDirs.length) : submissionDirs.length;
            // fromSubmissionChecked is inclusive
            let fromSubmissionChecked = fromSubmission > 0 ? Math.min(fromSubmission - 1, submissionDirs.length - 1) : 0;
            // if to/from not specified: submissionDirs.length-0 = submissionDirs.length
            let countFromToTo = toSubmissionChecked - fromSubmissionChecked;

            let maxCheckedSubmissions = max > 0 ? Math.min(max, countFromToTo) : countFromToTo;

            if (selected.length > 0 && max > 0) {
                log(`Filter submissions with ids ${selected.join(', ')}, ignore max value.`);
                maxCheckedSubmissions = submissionDirs.length;
            }
            if (selectPatched) {
                if (selected.length > 0) {
                    warn(`--selectPatched and --selected are mutually exclusive, ignoring --selectPatched.`);
                } else {
                    const patchFiles = await readDir(patchFolder, 'file', false, 'Check patch folder.');
                    patchedSubmissions.push(... patchFiles.map(f => {
                        const n_s = f.substring(0, f.lastIndexOf('.')).split('_');
                        return { name: n_s[0], submissionId: n_s[1] };
                    }));
                    verb(`Found ${patchFiles.length} patch files: ${patchedSubmissions.map(s=>s.name).join(', ')}.`)
                }

            }
            verb(`Found ${submissionDirs.length} submissions, process from ${fromSubmissionChecked} to ${toSubmissionChecked} (max. ${maxCheckedSubmissions}).`);
            for (let counter = fromSubmissionChecked; counter < fromSubmissionChecked + maxCheckedSubmissions; counter++) {
                try {
                    const subDir = submissionDirs[counter];
                    ({ name, submissionId: subId } = parseSubmitter(subDir));
                    if (!isSubmissionSelected(selected, patchedSubmissions, name, subId)) {
                        verb(`Skip user ${name}, submission id ${subId}`);
                        continue;
                    }
                    const absSubDir = path.join(submissionsDir, subDir);
                    log(SEP);
                    log(`\nProcessing submission ${counter + 1} of ${submissionDirs.length}: ${subId} ${name}\n`, options.colorSection);

                    // find the submitted zip file:
                    let submittedFiles = await readDir(absSubDir, 'file');
                    if (submittedFiles.length != 1) {
                        log(`Warning: ${name} submitted ${submittedFiles.length} files`);
                        if (submittedFiles.length > 1) {
                            submittedFiles = submittedFiles.filter(f => f.endsWith('.zip'));
                            if (submittedFiles.length != 1) {
                                log(`Warning: ${name} submitted ${submittedFiles.length} zip-files, take first one.`);
                            }
                            submittedFiles = [submittedFiles[0]]
                        }
                        if (submittedFiles.length != 1) {
                            log(`Warning: Skipping submission of ${name}.`);
                            continue;
                        }
                    }
                    // unzip the submitted zip file
                    const wsDir = path.join(submissionsDir, subDir, 'ws' + (name ? "_" + name.replace(/ /g, '_') : ""));
                    if (!skipPrepareSubmission) {
                        if (await folderExists(wsDir)) {
                            log(`Remove workspace folder ${wsDir}.`)
                            await rmDir(wsDir);
                        }
                        await createDir(wsDir);
                        const subZip = path.join(absSubDir, submittedFiles[0])
                        await unzip(subZip, wsDir);
                    } else {
                        if (! await folderExists(wsDir, 'workspace folder')) {
                            warn(`Skipping submission of ${name}, workspace folder not unzipped yet and skipPrepareSubmission is set.`);
                            continue;
                        }
                    }

                    // find project folder, usually the one folder contained in the submission
                    const projectDir = await findDirContainingFile(wsDir, options.projectFile, options.ignoredDirs);
                    if (prepareSubmissionCmd && !skipPrepareSubmission) {
                        // usually git reset --hard
                        await execShell(prepareSubmissionCmd.cmd, prepareSubmissionCmd.args,
                            { ...options, colored: options.colored || false, colorStandard: options.colorPre }, { indent: false, prefix: 'pre: ', cwd: projectDir });

                        if (patch && await hasLocalGitRepository(projectDir)) {
                            const patchFileName = getPatchFileName(patchFolder, name, subId);
                            if (await fileExists(patchFileName, 'patch file')) {
                                if (dry) {
                                    log(`Dry run: Would checkout new branch ${patchGradingBranch}, apply patch (and commit), and work on that branch.`);
                                } else {
                                    log(`Patch file ${patchFileName} exists, will checkout new branch ${patchGradingBranch}, apply patch (and commit), and work on that branch.`);
                                    try {
                                        await gitCreateAndCheckoutBranch(projectDir, patchGradingBranch, { colored: options.colored || false });
                                        await gitApplyPatch(projectDir, patchFileName, { colored: options.colored || false });
                                        await gitAddAndCommit(projectDir, `Applied lecturer's patch`, { colored: options.colored || false });
                                    } catch (err) {
                                        warn(`Error during patching: ${err}`);
                                    }
                                }
                            }
                        }


                    }

                    if (!onlyPrepareSubmission) {
                        const clipUserDir = await createClipSubmissionDir(clipDir, subId);
                        await runPreCheckScript(options.preCheckScript, projectDir, clipUserDir, checkDir, false, options);
                        try {
                            await runDocker("check", skipStudentTests, checkDir, projectDir, reportsDir, npmCacheDir, cacheDir, clipUserDir, volumes, subId,
                                {
                                    dry: options.dry, dockerImage: options.dockerImage, 
                                    dockerUserDir: options.dockerUserDir,
                                    dockerWorkDir: options.dockerWorkDir,
                                    dockerShellCmd: options.dockerShellCmd, dockerArgs: options.dockerArgs,
                                    dockerScript: options.dockerScript, timeoutPerDockerRun: options.timeoutPerDockerRun,
                                    preserveContainer: false
                                }, options
                            );
                        } finally {
                            await runPostCheckScript(options.postCheckScript, projectDir, clipUserDir, checkDir, false, options);
                            if (clipUserDir) {
                                await rmClipSubmissionDir(clipUserDir, subId);
                            }
                        }
                    }

                } catch (err) {
                    const msg = `${name}: ${err instanceof Error ? err.message : err}`
                    error(`${msg}`)
                    error(`Continue with next submission.`)
                    errors.push(msg);
                }


            }
            if (submissionDirs.length > maxCheckedSubmissions) {
                log(`Maximal submissions reached, skipping ${submissionDirs.length - maxCheckedSubmissions} submissions.`)
            }

        } catch (err) {
            error(`${SEP}`);
            if (errors.length > 0) {
                error(`There were ${errors.length} errors:`)
                errors.forEach(msg => error(msg));
            }
            program.error(String(err));
        }
        const endTime = hrtime.bigint();
        const duration = timeDiff(startTime, endTime)
        if (!runInternally) {
            log(`${SEP}\nDone, needed ${duration} minutes.`);
            if (errors.length > 0) {
                error(`There were ${errors.length} errors:`)
                errors.forEach(msg => error(msg));
            }
        }
    } finally {
        console.log = orgConsoleLog;
        console.error = orgConsoleError;
        console.warn = orgConsoleWarn;
    }
}

type PhaseType = "prepare" | "check";

export async function runDocker(phase: PhaseType, skipStudentTests: boolean, checkDir: string, projectDir: string,
    reportsDir: string, npmCacheDir: string, cacheDir: string, clipUserDir: string | undefined, volumes: string[], submissionId: string,
    optionsForDocker: {
        dry: boolean, dockerImage: string, 
        dockerUserDir: string, 
        dockerWorkDir: string, 
        dockerScript: string, dockerArgs: string[]
        timeoutPerDockerRun: number, dockerShellCmd: string
        preserveContainer: boolean
    }, options: OptionValues) {
    const dockerArgs = [`run`]
    if (!optionsForDocker.preserveContainer) dockerArgs.push(`--rm`);
    dockerArgs.push(
        `-v`, `${absPath(checkDir)}/:/check`,
        `-v`, `${absPath(projectDir)}/:/testee`,
        `-v`, `${absPath(reportsDir)}/:/reports`
    );
    let userDir = optionsForDocker.dockerUserDir;
    if (!userDir.startsWith('/')) userDir = '/' + userDir;
    if (!userDir.endsWith('/')) userDir = userDir + '/';
    let workDir = optionsForDocker.dockerWorkDir;
    if (!workDir.startsWith('/')) workDir = '/' + workDir;
    if (npmCacheDir) dockerArgs.push(`-v`, `${absPath(npmCacheDir)}/:${userDir}.npm`);
    if (cacheDir) dockerArgs.push(`-v`, `${absPath(cacheDir)}/:${userDir}.cache`);
    if (clipUserDir) dockerArgs.push(`-v`, `${absPath(clipUserDir)}/:/clip`);
    if (volumes) {
        await Promise.all(
            volumes.map(async (v) => {
                const parts = v.split(':');
                if (parts.length != 2) {
                    program.error(`Invalid volume ${v}, must be hostDir:containerDir. It must contain on colon only!`);
                }
                let [host, container] = parts;
                if (!host.startsWith('/')) {
                    const relHost = host;
                    host = path.resolve(host);
                    verb(`Made mounted folder absolute: ${relHost} --> ${host}`)
                    await createDir(host);
                }
                verb(`Add volume ${host}:${container}`);
                dockerArgs.push(`-v`, `${host}:${container}`);
            }))
    };
    if (optionsForDocker.dockerArgs) dockerArgs.push(...optionsForDocker.dockerArgs);
    dockerArgs.push(
        `-e`, `SUBMISSIONID=${submissionId}`,
        `-e`, `PHASE=${phase}`,
        `-e`, `SKIP_STUDENT_TESTS=${skipStudentTests ? "true" : "false"}`,
        `-w`, `${workDir}`,
        `--name`, `container${submissionId}`,
        `${optionsForDocker.dockerImage}`,
        `timeout`, `${optionsForDocker.timeoutPerDockerRun}`,
        `${optionsForDocker.dockerShellCmd}`, `${optionsForDocker.dockerScript}`);
    log("docker " + dockerArgs.join(' '));

    if (optionsForDocker.dry) {
        log("Dry run, skip actual docker call.");
    } else {
        const startDockerTime = hrtime.bigint();

        const returnCode = await execShell("docker", dockerArgs, { ...options, colored: options.colored || false });

        const endDockerTime = hrtime.bigint();
        const dockerDuration = timeDiff(startDockerTime, endDockerTime);
        if (returnCode===124) {
            warn(`Docker run timed out after ${optionsForDocker.timeoutPerDockerRun} seconds.`);
        }
        log(`Check needed ${dockerDuration} minutes.`);
        await ensureFileExists(path.join(reportsDir, `_${submissionId}_junit.check.xml`));

    }
}


export function parsePrepareSubmissionCmd(prepareSubmissionCmd?: string) {
    if (prepareSubmissionCmd && prepareSubmissionCmd.length > 0) {
        const all = prepareSubmissionCmd.split(' ');
        const cmd = all.shift();
        if (!cmd) {
            program.error(`prepareSubmissionCmd must start with the command to execute.`);
        }
        return { cmd: cmd, args: all };
    } else {
        return undefined;
    }
}
export async function createClipSubmissionDir(clipDir: any, submissionId: string) {
    if (clipDir) {
        const clipUserDir = path.join(clipDir, submissionId);
        if (await folderExists(clipUserDir)) {
            await rmDir(clipUserDir);
        }
        await createDir(clipUserDir);
        return clipUserDir;
    }
    return undefined;
}

export async function rmClipSubmissionDir(clipUserDir: any, submissionId: string) {
    if (clipUserDir) {
        if (await isEmptyDir(clipUserDir)) {
            await rmDir(clipUserDir);
        }
    }
}
export async function runPreCheckScript(preCheckScript: string, projectDir: string, clipUserDir: string | undefined, checkDir: string, prepare: boolean, options: OptionValues) {
    if (preCheckScript) {
        await runPrePostCheckScript("pre", preCheckScript, projectDir, clipUserDir, checkDir, prepare, options);
    }
}
export async function runPostCheckScript(postCheckScript: string, projectDir: string, clipUserDir: string | undefined, checkDir: string, prepare: boolean, options: OptionValues) {
    if (postCheckScript) {
        await runPrePostCheckScript("post", postCheckScript, projectDir, clipUserDir, checkDir, prepare, options);
    }
}

async function runPrePostCheckScript(phase: "pre" | "post", checkScript: string, projectDir: string, clipUserDir: string | undefined, checkDir: string, prepare: boolean, options: OptionValues) {

    if (await fileExists(checkScript, `name of ${phase}CheckScript not provided`)) {

        const env: NodeJS.ProcessEnv = {
            ...process.env,
            CHECK: checkDir,
            PROJECT: projectDir
        }
        if (clipUserDir) {
            env.CLIP = clipUserDir;
        }
        if (prepare) {
            env.PREPARE = "1";
        } else {
            env.PREPARE = "0";
        }


        await execShell("bash", [checkScript],
            { ...options, colored: options.colored || false, colorStandard: options.colorPost },
            {
                indent: true,
                prefix: `${phase}> `,
                cwd: undefined, // use current working directory
                env: env
            });
    }
}