grading
Version:
Grading of student submissions, in particular programming tests.
440 lines • 18 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.CSVConsumer = exports.CSVProducer = exports.parseGrading = exports.writeTableWithEncoding = exports.loadTableWithEncoding = exports.readCSV = exports.parseCSV = void 0;
const fs_1 = __importDefault(require("fs"));
const stream_1 = require("stream");
const promises_1 = require("stream/promises");
const table_1 = require("./table");
const iconv = __importStar(require("iconv-lite"));
/**
* Parses an CSV string.
*/
async function parseCSV(csv, delimiter = ',') {
const readable = stream_1.Readable.from(csv, { objectMode: false, highWaterMark: 1 });
return readCSV(readable, delimiter);
}
exports.parseCSV = parseCSV;
async function readCSV(readable, delimiter = ',') {
const table = new table_1.Table();
readable.setEncoding("utf-8");
let mode = "before";
let col = 1;
let row = 1;
let bufferedWS = "";
let value = "";
let c; // defined here to be avaiable in at()
const at = () => {
return " at row " + row + ", col " + col + ", last char: '" + c + "'";
};
readable.on("readable", () => {
try {
while ((c = readable.read(1)) != null) {
if (c.charCodeAt(0) > 40000) {
console.log("Skip weird character: '" + c + "', char code " + c.charCodeAt(0));
continue;
}
switch (mode) {
case "before":
switch (c) {
case " ":
case "\t":
// skip
break;
case '"':
mode = "escaped";
break;
case delimiter: // empty value
table.setText(col, row, value);
col++;
value = "";
break;
default:
value += c;
mode = "nonEscaped";
}
break;
case "escaped":
if (c === '"') {
mode = "dquoteInEscaped";
}
else {
value += c;
}
break;
case "dquoteInEscaped":
if (c == '"') { // 2 double quotes
value += '"';
mode = "escaped";
break;
}
else {
mode = "after";
// and fall through
}
case "after": // after escaped
switch (c) {
case " ":
case "\t":
case "\r":
// skip;
break;
case delimiter:
table.setText(col, row, value);
mode = "before";
col++;
value = "";
break;
case "\n":
table.setText(col, row, value);
row++;
col = 1;
value = "";
mode = "before";
break;
default:
throw new Error("Unexpected data after closing dquote" + at());
}
break;
case "nonEscaped":
switch (c) {
case " ":
case "\t":
bufferedWS += c;
break;
case delimiter:
table.setText(col, row, value);
col++;
value = "";
bufferedWS = "";
mode = "before";
break;
case "\r":
const next = readable.read(1);
if (next != null && next != "\n") {
throw new Error("Unexpected LF/CR" + at());
}
// and:
case "\n":
table.setText(col, row, value);
row++;
col = 1;
value = "";
mode = "before";
break;
case '"':
throw new Error("Strings with DQUOTEs must be escaped" + at());
default:
if (bufferedWS.length > 0) {
value += bufferedWS;
bufferedWS = "";
}
value += c;
}
}
} // end while (no more data)
if (value.length > 0) {
table.setText(col, row, value);
}
}
catch (e) {
readable.destroy(e); // instead of throw!
}
});
await (0, promises_1.finished)(readable);
return table;
}
exports.readCSV = readCSV;
const DEFAULT_ENCODING = "utf-8";
async function loadTableWithEncoding(pathName, encoding, delimiter) {
const csvStream = new CSVConsumer({ delimiter: delimiter });
if (encoding.toLowerCase() === DEFAULT_ENCODING) {
const readstream = fs_1.default.createReadStream(pathName, { encoding: DEFAULT_ENCODING });
await (0, promises_1.pipeline)(readstream, csvStream);
}
else {
const readstream = fs_1.default.createReadStream(pathName);
await (0, promises_1.pipeline)(readstream, iconv.decodeStream(encoding), csvStream);
}
const res = csvStream.getTable();
return res;
}
exports.loadTableWithEncoding = loadTableWithEncoding;
async function writeTableWithEncoding(pathName, table, encoding, delimiter) {
const writeStream = fs_1.default.createWriteStream(pathName, { encoding: DEFAULT_ENCODING });
const csvStream = new CSVProducer(table, { delimiter: delimiter, encoding: DEFAULT_ENCODING });
if (encoding.toLowerCase() === "utf-8") {
await (0, promises_1.pipeline)(csvStream, writeStream);
}
else {
await (0, promises_1.pipeline)(csvStream, iconv.encodeStream(encoding), writeStream);
}
}
exports.writeTableWithEncoding = writeTableWithEncoding;
function parseGrading(gradingWithComma) {
return parseFloat(gradingWithComma.replace(",", "."));
}
exports.parseGrading = parseGrading;
class CSVProducer extends stream_1.Readable {
constructor(table, csvOptions) {
super(csvOptions);
this.col = 1;
this.row = 1;
this.table = table;
//super({...options, decodeStrings:true}); // use spread operator
this.delimiter = csvOptions?.delimiter || ",";
this.escapeAlways = csvOptions?.escapeAlways || false;
this.escapeWhenSpace = csvOptions?.escapeWhenSpace || false;
this.lineBreak = csvOptions?.lineBreak || "\n";
}
_read(_size) {
if (this.row > this.table.rowsCount) {
this.push(null);
}
else {
let chunk = "";
while (this.row <= this.table.rowsCount) {
const field = this.table.getText(this.col, this.row);
if (this.col > 1 && this.col <= this.table.columnsCount) {
chunk += this.delimiter;
}
chunk += this.escapeIfNecessary(field);
this.col++;
if (this.col > this.table.columnsCount) {
this.col = 1;
this.row++;
if (this.row <= this.table.rowsCount) {
chunk += "\n";
}
}
}
this.push(chunk);
}
}
escapeIfNecessary(field) {
let maskedField = "";
let escapedNecessary = this.escapeAlways;
for (let i = 0; i < field.length; i++) {
const c = field.charAt(i);
maskedField += c;
switch (c) {
case '"':
maskedField += '"';
escapedNecessary = true;
break;
case " ":
if (!this.escapeWhenSpace) {
break;
}
case this.delimiter:
case "\n":
case "\r":
escapedNecessary = true;
break;
}
}
if (!escapedNecessary) {
if (field.length > 0 && " \t".indexOf(field.charAt(0)) >= 0) {
escapedNecessary = true;
}
else if (field.length > 0 && " \t".indexOf(field.charAt(field.length - 1)) >= 0) {
escapedNecessary = true;
}
}
if (escapedNecessary) {
maskedField = '"' + maskedField + '"';
}
return maskedField;
}
}
exports.CSVProducer = CSVProducer;
/**
* Klasse zum Einlesen von Tabellen im Format CSV (Comma Separated
* Values).
* <p>
* Dieses Format ist (hier) wie folgt definiert:
*
* <ol>
* <li>Jede Tabellenzeile ist eine Textzeile in der Datei, die Zeilen sind mit "\n" getrennt.
* <li>Die Tabellenzellen sind mit ";" voneinander getrennt.
* <li>Der Tabelleninhalt darf alle Zeichen außer ";", "\n" oder "\r" enthalten.
* <li>Leerzeichen direkt vor oder nach dem ";" werden ignoriert.
* <li>Alle Zeilen müssen die gleiche Anzahl an Spalten haben.
* <li>Leere Tabellenzellen sind möglich
* </ol>
*
* @author Jens von Pilgrim
*/
class CSVConsumer extends stream_1.Writable {
at() {
return " at row " + this.row + ", col " + this.col + ", last char: '" + this.c + "'";
}
constructor(options) {
super(options);
this.table = new table_1.Table();
this.mode = "before";
this.col = 1;
this.row = 1;
this.bufferedWS = "";
this.value = "";
this.c = null; // defined here to be avaiable in at()
//super({...options, decodeStrings:true}); // use spread operator
this.delimiter = options?.delimiter || ",";
}
_write(chunk, _encoding, callback) {
if (chunk == null) {
callback();
return;
}
const s = chunk.toString();
// callback(new Error("just because")); return;
try {
for (this.c of s) {
if (this.c.charCodeAt(0) > 40000) {
console.log("Skip weird character: '" + this.c + "', char code " + this.c.charCodeAt(0));
continue;
}
switch (this.mode) {
case "before":
switch (this.c) {
case " ":
case "\t":
// skip
break;
case '"':
this.mode = "escaped";
break;
case this.delimiter: // empty value
this.table.setText(this.col, this.row, this.value);
this.col++;
this.value = "";
break;
case "\n":
this.table.setText(this.col, this.row, this.value);
this.row++;
this.col = 1;
this.value = "";
this.mode = "before";
break;
default:
this.value += this.c;
this.mode = "nonEscaped";
}
break;
case "escaped":
if (this.c === '"') {
this.mode = "dquoteInEscaped";
}
else {
this.value += this.c;
}
break;
case "dquoteInEscaped":
if (this.c == '"') { // 2 double quotes
this.value += '"';
this.mode = "escaped";
break;
}
else {
this.mode = "after";
// and fall through
}
case "after": // after escaped
switch (this.c) {
case " ":
case "\t":
case "\r":
// skip;
break;
case this.delimiter:
this.table.setText(this.col, this.row, this.value);
this.mode = "before";
this.col++;
this.value = "";
break;
case "\n":
this.table.setText(this.col, this.row, this.value);
this.row++;
this.col = 1;
this.value = "";
this.mode = "before";
break;
default:
throw new Error("Unexpected data after closing dquote" + this.at());
}
break;
case "nonEscaped":
switch (this.c) {
case " ":
case "\t":
case "\r":
this.bufferedWS += this.c;
break;
case this.delimiter:
this.table.setText(this.col, this.row, this.value);
this.col++;
this.value = "";
this.bufferedWS = "";
this.mode = "before";
break;
case "\n":
this.table.setText(this.col, this.row, this.value);
this.row++;
this.col = 1;
this.value = "";
this.mode = "before";
break;
case '"':
throw new Error("Strings with DQUOTEs must be escaped" + this.at());
default:
if (this.bufferedWS.length > 0) {
this.value += this.bufferedWS;
this.bufferedWS = "";
}
this.value += this.c;
}
}
} // end for
if (this.value.length > 0 ||
(this.mode == "before" && this.col > 1) // or we had an empty field before (e.g. ",,")
) {
this.table.setText(this.col, this.row, this.value);
}
callback(); // do not forget to call this, otherwise pipeline does not know when to finish
}
catch (e) {
callback(e); // pipeline will re-throw the error in await case, very nice!
}
}
getTable() {
this.destroy();
return this.table;
}
}
exports.CSVConsumer = CSVConsumer;
//# sourceMappingURL=csv.js.map