grading
Version:
Grading of student submissions, in particular programming tests.
434 lines • 23.4 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.runPostCheckScript = exports.runPreCheckScript = exports.rmClipSubmissionDir = exports.createClipSubmissionDir = exports.parsePrepareSubmissionCmd = exports.runDocker = exports.cmdCheck = exports.cmdCheckAction = void 0;
const fs = __importStar(require("fs"));
const path_1 = __importDefault(require("path"));
const process_1 = require("process");
const fsUtil_1 = require("../fsUtil");
const gitCommands_1 = require("../gitCommands");
const cliUtil_1 = require("./cliUtil");
async function cmdCheckAction(zipFile, options) {
return await cmdCheck(zipFile, options);
}
exports.cmdCheckAction = cmdCheckAction;
function appendLog(logfile, message, ...optionalParams) {
let out = message ?? "";
if (optionalParams) {
out += " " + optionalParams.join(" ");
}
out += "\n";
fs.appendFileSync(logfile, out, { encoding: "utf-8" });
}
async function rewireConsoleOutput(logOutput) {
if (!logOutput) {
return;
}
const logFile = (0, cliUtil_1.generateFileName)(logOutput, {
dateTime: Date.now()
});
if (await (0, fsUtil_1.fileExists)(logFile)) {
(0, cliUtil_1.log)(`Reset log file ${logFile}`);
(0, fsUtil_1.rmFile)(logFile);
}
const parentFolder = path_1.default.dirname(logFile);
if (!await (0, fsUtil_1.fileExists)(parentFolder)) {
await (0, fsUtil_1.createDir)(parentFolder);
}
const orgConsoleLog = console.log;
const orgConsoleError = console.error;
const orgConsoleWarn = console.warn;
console.log = function (message, ...optionalParams) {
orgConsoleLog(message, ...optionalParams);
appendLog(logFile, message, ...optionalParams);
};
console.error = function (message, ...optionalParams) {
orgConsoleError(message, ...optionalParams);
appendLog(logFile, message, ...optionalParams);
};
console.warn = function (message, ...optionalParams) {
orgConsoleWarn(message, ...optionalParams);
appendLog(logFile, message, ...optionalParams);
};
orgConsoleLog(`Write all output to ${logFile}`);
}
async function cmdCheck(zipFile, options, runInternally = false, errors) {
(0, cliUtil_1.verbosity)(options);
if (!errors) {
errors = [];
}
const orgConsoleLog = console.log;
const orgConsoleError = console.error;
const orgConsoleWarn = console.warn;
try {
const logOutput = options.logOutput;
await rewireConsoleOutput(logOutput);
const startTime = process_1.hrtime.bigint();
if (runInternally) {
(0, cliUtil_1.log)("Start running tests on submissions.");
}
else {
(0, cliUtil_1.log)("Start running tests on submissions at " + (0, fsUtil_1.timestamp)());
}
try {
zipFile = await (0, cliUtil_1.retrieveZipFile)(zipFile, options);
await (0, fsUtil_1.ensureFileExists)(zipFile, "Check zip file.");
const submissionsDir = options.submissionsDir;
const checkDir = options.checkDir;
const clipDir = options.clipDir;
const reportsDir = options.reportsDir;
const npmCacheDir = options.npmCacheDir;
const cacheDir = options.cacheDir;
const dry = options.dry;
const prepareSubmissionCmd = parsePrepareSubmissionCmd(options.prepareSubmissionCmd);
const onlyPrepareSubmission = options.onlyPrepareSubmission;
const skipPrepareSubmission = options.skipPrepareSubmission;
const volumes = options.volumes;
const skipStudentTests = options.skipStudentTests;
const patch = options.patch;
const patchFolder = options.patchFolder;
const patchGradingBranch = options.patchGradingBranch;
if (onlyPrepareSubmission && skipPrepareSubmission) {
program.error(`--skipPrepareSubmission and --onlyPrepareSubmission are mutually exclusive.`);
}
if (onlyPrepareSubmission && !prepareSubmissionCmd) {
(0, cliUtil_1.warn)(`--onlyPrepareSubmission is set, but no prepareSubmissionCmd is defined. Nothing to do.`);
}
const selected = (0, cliUtil_1.retrieveSelectedFilter)(options);
const selectPatched = options.selectPatched;
const patchedSubmissions = [];
const cleanBefore = options.cleanBefore;
if (!cleanBefore) {
(0, cliUtil_1.verb)('Do not clean before.');
}
const max = Number.parseInt(options.max);
const fromSubmission = Number.parseInt(options.fromSubmission);
const toSubmission = Number.parseInt(options.toSubmission);
await (0, fsUtil_1.ensureFolderExists)(checkDir, "Check check folder.");
if (npmCacheDir && !await (0, fsUtil_1.folderExists)(npmCacheDir, 'npmCacheDir'))
await (0, fsUtil_1.createDir)(npmCacheDir);
if (cacheDir && !await (0, fsUtil_1.folderExists)(cacheDir, 'cacheDir'))
await (0, fsUtil_1.createDir)(cacheDir);
if (clipDir && !await (0, fsUtil_1.folderExists)(clipDir, 'clipDir'))
await (0, fsUtil_1.createDir)(clipDir);
if (cleanBefore && await (0, fsUtil_1.folderExists)(submissionsDir, 'submissionsDir')) {
(0, cliUtil_1.verb)(`Remove submissions folder ${submissionsDir} (clean).`);
await (0, fsUtil_1.rmDir)(submissionsDir);
}
if (!await (0, fsUtil_1.folderExists)(submissionsDir, 'folderExists'))
await (0, fsUtil_1.createDir)(submissionsDir);
if (cleanBefore && !dry && !skipStudentTests && await (0, fsUtil_1.folderExists)(reportsDir, 'reportsDir'))
await (0, fsUtil_1.rmDir)(reportsDir); // don't clear on dry run
if (!await (0, fsUtil_1.folderExists)(reportsDir, 'reportsDir'))
await (0, fsUtil_1.createDir)(reportsDir);
await (0, fsUtil_1.unzip)(zipFile, submissionsDir); // we always unzip the big file
const submissionDirs = await (0, fsUtil_1.readDir)(submissionsDir, 'dir', false, 'Check submissions folder.');
if (submissionDirs.length == 0) {
program.error(`No submissions found in ${submissionsDir}`);
}
let name = "", subId = ""; // define here for catch block
// toSubmission and fromSubmission are 1-based
// toSubmissionChecked is exclusive
let toSubmissionChecked = toSubmission > 0 ? Math.min(toSubmission, submissionDirs.length) : submissionDirs.length;
// fromSubmissionChecked is inclusive
let fromSubmissionChecked = fromSubmission > 0 ? Math.min(fromSubmission - 1, submissionDirs.length - 1) : 0;
// if to/from not specified: submissionDirs.length-0 = submissionDirs.length
let countFromToTo = toSubmissionChecked - fromSubmissionChecked;
let maxCheckedSubmissions = max > 0 ? Math.min(max, countFromToTo) : countFromToTo;
if (selected.length > 0 && max > 0) {
(0, cliUtil_1.log)(`Filter submissions with ids ${selected.join(', ')}, ignore max value.`);
maxCheckedSubmissions = submissionDirs.length;
}
if (selectPatched) {
if (selected.length > 0) {
(0, cliUtil_1.warn)(`--selectPatched and --selected are mutually exclusive, ignoring --selectPatched.`);
}
else {
const patchFiles = await (0, fsUtil_1.readDir)(patchFolder, 'file', false, 'Check patch folder.');
patchedSubmissions.push(...patchFiles.map(f => {
const n_s = f.substring(0, f.lastIndexOf('.')).split('_');
return { name: n_s[0], submissionId: n_s[1] };
}));
(0, cliUtil_1.verb)(`Found ${patchFiles.length} patch files: ${patchedSubmissions.map(s => s.name).join(', ')}.`);
}
}
(0, cliUtil_1.verb)(`Found ${submissionDirs.length} submissions, process from ${fromSubmissionChecked} to ${toSubmissionChecked} (max. ${maxCheckedSubmissions}).`);
for (let counter = fromSubmissionChecked; counter < fromSubmissionChecked + maxCheckedSubmissions; counter++) {
try {
const subDir = submissionDirs[counter];
({ name, submissionId: subId } = (0, cliUtil_1.parseSubmitter)(subDir));
if (!(0, cliUtil_1.isSubmissionSelected)(selected, patchedSubmissions, name, subId)) {
(0, cliUtil_1.verb)(`Skip user ${name}, submission id ${subId}`);
continue;
}
const absSubDir = path_1.default.join(submissionsDir, subDir);
(0, cliUtil_1.log)(SEP);
(0, cliUtil_1.log)(`\nProcessing submission ${counter + 1} of ${submissionDirs.length}: ${subId} ${name}\n`, options.colorSection);
// find the submitted zip file:
let submittedFiles = await (0, fsUtil_1.readDir)(absSubDir, 'file');
if (submittedFiles.length != 1) {
(0, cliUtil_1.log)(`Warning: ${name} submitted ${submittedFiles.length} files`);
if (submittedFiles.length > 1) {
submittedFiles = submittedFiles.filter(f => f.endsWith('.zip'));
if (submittedFiles.length != 1) {
(0, cliUtil_1.log)(`Warning: ${name} submitted ${submittedFiles.length} zip-files, take first one.`);
}
submittedFiles = [submittedFiles[0]];
}
if (submittedFiles.length != 1) {
(0, cliUtil_1.log)(`Warning: Skipping submission of ${name}.`);
continue;
}
}
// unzip the submitted zip file
const wsDir = path_1.default.join(submissionsDir, subDir, 'ws' + (name ? "_" + name.replace(/ /g, '_') : ""));
if (!skipPrepareSubmission) {
if (await (0, fsUtil_1.folderExists)(wsDir)) {
(0, cliUtil_1.log)(`Remove workspace folder ${wsDir}.`);
await (0, fsUtil_1.rmDir)(wsDir);
}
await (0, fsUtil_1.createDir)(wsDir);
const subZip = path_1.default.join(absSubDir, submittedFiles[0]);
await (0, fsUtil_1.unzip)(subZip, wsDir);
}
else {
if (!await (0, fsUtil_1.folderExists)(wsDir, 'workspace folder')) {
(0, cliUtil_1.warn)(`Skipping submission of ${name}, workspace folder not unzipped yet and skipPrepareSubmission is set.`);
continue;
}
}
// find project folder, usually the one folder contained in the submission
const projectDir = await (0, fsUtil_1.findDirContainingFile)(wsDir, options.projectFile, options.ignoredDirs);
if (prepareSubmissionCmd && !skipPrepareSubmission) {
// usually git reset --hard
await (0, fsUtil_1.execShell)(prepareSubmissionCmd.cmd, prepareSubmissionCmd.args, { ...options, colored: options.colored || false, colorStandard: options.colorPre }, { indent: false, prefix: 'pre: ', cwd: projectDir });
if (patch && await (0, gitCommands_1.hasLocalGitRepository)(projectDir)) {
const patchFileName = (0, gitCommands_1.getPatchFileName)(patchFolder, name, subId);
if (await (0, fsUtil_1.fileExists)(patchFileName, 'patch file')) {
if (dry) {
(0, cliUtil_1.log)(`Dry run: Would checkout new branch ${patchGradingBranch}, apply patch (and commit), and work on that branch.`);
}
else {
(0, cliUtil_1.log)(`Patch file ${patchFileName} exists, will checkout new branch ${patchGradingBranch}, apply patch (and commit), and work on that branch.`);
try {
await (0, gitCommands_1.gitCreateAndCheckoutBranch)(projectDir, patchGradingBranch, { colored: options.colored || false });
await (0, gitCommands_1.gitApplyPatch)(projectDir, patchFileName, { colored: options.colored || false });
await (0, gitCommands_1.gitAddAndCommit)(projectDir, `Applied lecturer's patch`, { colored: options.colored || false });
}
catch (err) {
(0, cliUtil_1.warn)(`Error during patching: ${err}`);
}
}
}
}
}
if (!onlyPrepareSubmission) {
const clipUserDir = await createClipSubmissionDir(clipDir, subId);
await runPreCheckScript(options.preCheckScript, projectDir, clipUserDir, checkDir, false, options);
try {
await runDocker("check", skipStudentTests, checkDir, projectDir, reportsDir, npmCacheDir, cacheDir, clipUserDir, volumes, subId, {
dry: options.dry, dockerImage: options.dockerImage,
dockerUserDir: options.dockerUserDir,
dockerWorkDir: options.dockerWorkDir,
dockerShellCmd: options.dockerShellCmd, dockerArgs: options.dockerArgs,
dockerScript: options.dockerScript, timeoutPerDockerRun: options.timeoutPerDockerRun,
preserveContainer: false
}, options);
}
finally {
await runPostCheckScript(options.postCheckScript, projectDir, clipUserDir, checkDir, false, options);
if (clipUserDir) {
await rmClipSubmissionDir(clipUserDir, subId);
}
}
}
}
catch (err) {
const msg = `${name}: ${err instanceof Error ? err.message : err}`;
(0, cliUtil_1.error)(`${msg}`);
(0, cliUtil_1.error)(`Continue with next submission.`);
errors.push(msg);
}
}
if (submissionDirs.length > maxCheckedSubmissions) {
(0, cliUtil_1.log)(`Maximal submissions reached, skipping ${submissionDirs.length - maxCheckedSubmissions} submissions.`);
}
}
catch (err) {
(0, cliUtil_1.error)(`${SEP}`);
if (errors.length > 0) {
(0, cliUtil_1.error)(`There were ${errors.length} errors:`);
errors.forEach(msg => (0, cliUtil_1.error)(msg));
}
program.error(String(err));
}
const endTime = process_1.hrtime.bigint();
const duration = (0, cliUtil_1.timeDiff)(startTime, endTime);
if (!runInternally) {
(0, cliUtil_1.log)(`${SEP}\nDone, needed ${duration} minutes.`);
if (errors.length > 0) {
(0, cliUtil_1.error)(`There were ${errors.length} errors:`);
errors.forEach(msg => (0, cliUtil_1.error)(msg));
}
}
}
finally {
console.log = orgConsoleLog;
console.error = orgConsoleError;
console.warn = orgConsoleWarn;
}
}
exports.cmdCheck = cmdCheck;
async function runDocker(phase, skipStudentTests, checkDir, projectDir, reportsDir, npmCacheDir, cacheDir, clipUserDir, volumes, submissionId, optionsForDocker, options) {
const dockerArgs = [`run`];
if (!optionsForDocker.preserveContainer)
dockerArgs.push(`--rm`);
dockerArgs.push(`-v`, `${(0, fsUtil_1.absPath)(checkDir)}/:/check`, `-v`, `${(0, fsUtil_1.absPath)(projectDir)}/:/testee`, `-v`, `${(0, fsUtil_1.absPath)(reportsDir)}/:/reports`);
let userDir = optionsForDocker.dockerUserDir;
if (!userDir.startsWith('/'))
userDir = '/' + userDir;
if (!userDir.endsWith('/'))
userDir = userDir + '/';
let workDir = optionsForDocker.dockerWorkDir;
if (!workDir.startsWith('/'))
workDir = '/' + workDir;
if (npmCacheDir)
dockerArgs.push(`-v`, `${(0, fsUtil_1.absPath)(npmCacheDir)}/:${userDir}.npm`);
if (cacheDir)
dockerArgs.push(`-v`, `${(0, fsUtil_1.absPath)(cacheDir)}/:${userDir}.cache`);
if (clipUserDir)
dockerArgs.push(`-v`, `${(0, fsUtil_1.absPath)(clipUserDir)}/:/clip`);
if (volumes) {
await Promise.all(volumes.map(async (v) => {
const parts = v.split(':');
if (parts.length != 2) {
program.error(`Invalid volume ${v}, must be hostDir:containerDir. It must contain on colon only!`);
}
let [host, container] = parts;
if (!host.startsWith('/')) {
const relHost = host;
host = path_1.default.resolve(host);
(0, cliUtil_1.verb)(`Made mounted folder absolute: ${relHost} --> ${host}`);
await (0, fsUtil_1.createDir)(host);
}
(0, cliUtil_1.verb)(`Add volume ${host}:${container}`);
dockerArgs.push(`-v`, `${host}:${container}`);
}));
}
;
if (optionsForDocker.dockerArgs)
dockerArgs.push(...optionsForDocker.dockerArgs);
dockerArgs.push(`-e`, `SUBMISSIONID=${submissionId}`, `-e`, `PHASE=${phase}`, `-e`, `SKIP_STUDENT_TESTS=${skipStudentTests ? "true" : "false"}`, `-w`, `${workDir}`, `--name`, `container${submissionId}`, `${optionsForDocker.dockerImage}`, `timeout`, `${optionsForDocker.timeoutPerDockerRun}`, `${optionsForDocker.dockerShellCmd}`, `${optionsForDocker.dockerScript}`);
(0, cliUtil_1.log)("docker " + dockerArgs.join(' '));
if (optionsForDocker.dry) {
(0, cliUtil_1.log)("Dry run, skip actual docker call.");
}
else {
const startDockerTime = process_1.hrtime.bigint();
const returnCode = await (0, fsUtil_1.execShell)("docker", dockerArgs, { ...options, colored: options.colored || false });
const endDockerTime = process_1.hrtime.bigint();
const dockerDuration = (0, cliUtil_1.timeDiff)(startDockerTime, endDockerTime);
if (returnCode === 124) {
(0, cliUtil_1.warn)(`Docker run timed out after ${optionsForDocker.timeoutPerDockerRun} seconds.`);
}
(0, cliUtil_1.log)(`Check needed ${dockerDuration} minutes.`);
await (0, fsUtil_1.ensureFileExists)(path_1.default.join(reportsDir, `_${submissionId}_junit.check.xml`));
}
}
exports.runDocker = runDocker;
function parsePrepareSubmissionCmd(prepareSubmissionCmd) {
if (prepareSubmissionCmd && prepareSubmissionCmd.length > 0) {
const all = prepareSubmissionCmd.split(' ');
const cmd = all.shift();
if (!cmd) {
program.error(`prepareSubmissionCmd must start with the command to execute.`);
}
return { cmd: cmd, args: all };
}
else {
return undefined;
}
}
exports.parsePrepareSubmissionCmd = parsePrepareSubmissionCmd;
async function createClipSubmissionDir(clipDir, submissionId) {
if (clipDir) {
const clipUserDir = path_1.default.join(clipDir, submissionId);
if (await (0, fsUtil_1.folderExists)(clipUserDir)) {
await (0, fsUtil_1.rmDir)(clipUserDir);
}
await (0, fsUtil_1.createDir)(clipUserDir);
return clipUserDir;
}
return undefined;
}
exports.createClipSubmissionDir = createClipSubmissionDir;
async function rmClipSubmissionDir(clipUserDir, submissionId) {
if (clipUserDir) {
if (await (0, fsUtil_1.isEmptyDir)(clipUserDir)) {
await (0, fsUtil_1.rmDir)(clipUserDir);
}
}
}
exports.rmClipSubmissionDir = rmClipSubmissionDir;
async function runPreCheckScript(preCheckScript, projectDir, clipUserDir, checkDir, prepare, options) {
if (preCheckScript) {
await runPrePostCheckScript("pre", preCheckScript, projectDir, clipUserDir, checkDir, prepare, options);
}
}
exports.runPreCheckScript = runPreCheckScript;
async function runPostCheckScript(postCheckScript, projectDir, clipUserDir, checkDir, prepare, options) {
if (postCheckScript) {
await runPrePostCheckScript("post", postCheckScript, projectDir, clipUserDir, checkDir, prepare, options);
}
}
exports.runPostCheckScript = runPostCheckScript;
async function runPrePostCheckScript(phase, checkScript, projectDir, clipUserDir, checkDir, prepare, options) {
if (await (0, fsUtil_1.fileExists)(checkScript, `name of ${phase}CheckScript not provided`)) {
const env = {
...process.env,
CHECK: checkDir,
PROJECT: projectDir
};
if (clipUserDir) {
env.CLIP = clipUserDir;
}
if (prepare) {
env.PREPARE = "1";
}
else {
env.PREPARE = "0";
}
await (0, fsUtil_1.execShell)("bash", [checkScript], { ...options, colored: options.colored || false, colorStandard: options.colorPost }, {
indent: true,
prefix: `${phase}> `,
cwd: undefined,
env: env
});
}
}
//# sourceMappingURL=cmdCheck.js.map