grading

{ "$schema": "https://json-schema.org/draft-06/schema", "$id": "http://https://prof.bht-berlin.de/pilgrim/schemas/grading.schema.json", "$ref": "#/$defs/GradingSchema", "$defs": { "GradingSchema": { "description": "Definition of tasks and their points to automatically grade a submission.", "type": "object", "additionalProperties": false, "required": [ "course", "term", "exam", "points", "tasks" ], "properties": { "$schema": { "description": "URL of json schema, set to 'https://gitlab.bht-berlin.de/nodepackages/grading/-/raw/main/src/grade/grading.schema.json", "type": "string" }, "course": { "description": "Name of the course.", "type": "string" }, "term": { "description": "Term, e.g. WS 2022", "type": "string" }, "exam": { "description": "One exam is usually provided per week, this entry is also shown in the PDF later on.", "type": "string" }, "points": { "description": "Total number of points, must match sum of numbers for each task.", "type": "integer", "minimum": 0, "maximum": 1000 }, "minCoverageStatements": { "description": "Statement coverage that is required, usually 80 is used here. If not specified, coverage is ignored.", "type": "integer", "minimum": 0, "maximum": 100 }, "penaltiesCoverageMax": { "description": "Maximal points subtracted if coverage is below min coverage. Ignored if no minCoverageStatements is specified", "type": "integer", "minimum": 0, "maximum": 100 }, "extraCoverageMax": { "description": "Maximal points added if coverage exceeds min coverage. Ignored if no minCoverageStatements is specified", "type": "integer", "minimum": 0, "maximum": 100 }, "penalties": { "description": "Optional, if tests of penalties fail points are subtracted (well, actually added but the penalty points are expected to be negative). If the penalties have a precondition, the penalty is ignored if the precondition fails.", "type": "array", "items": { "$ref": "#/$defs/Penalty" } }, "tasks": { "description": "List of tasks and their grading.", "type": "array", "items": { "$ref": "#/$defs/Task" } } } }, "Task": { "description": "Grading of a single task, either grading or manual need to be set.", "type": "object", "additionalProperties": false, "required": [ "name", "points" ], "properties": { "name": { "description": "Name of the task.", "type": "string" }, "suite": { "description": "Test suite, this may be omitted if every grading defines its own suite. The suite name often is the name of the file with the tests.", "type": "string" }, "skipHint": { "description": "Text displayed if task cannot be graded, e.g., if precondition failed", "type": "string" }, "points": { "type": "integer", "minimum": 0, "maximum": 100 }, "preconditions": { "description": "Tests that define the precondition. If these tests fail, the grading is skipped. Order of preconditions matter (short circuit evaluation)!", "type": "array", "items": { "$ref": "#/$defs/Precondition" } }, "image": { "$ref": "#/$defs/Image", "description": "Deprecated, use images instead. Optional, if specified, the image is shown in the PDF if present." }, "images": { "description": "Optional, if specified, the images are shown in the PDF if present.", "type": "array", "items": { "$ref": "#/$defs/Image" } }, "grading": { "description": "Grading of the tasks by subtasks or aspects.", "type": "array", "items": { "$ref": "#/$defs/Grading" } }, "manual": { "description": "Manual grading only, no tests are executed.", "type": "boolean" }, "penalties": { "description": "Optional, if tests of penalties fail points are subtracted (well, actually added but the penalty points are expected to be negative). If the penalties have a precondition, the penalty is ignored if the precondition fails.", "type": "array", "items": { "$ref": "#/$defs/Penalty" } } } }, "Grading": { "description": "Grading (or penalty)", "type": "object", "additionalProperties": false, "required": [ "tests", "points" ], "properties": { "suite": { "description": "Test suite, this may be omitted if the test suite is defined in the task.", "type": "string" }, "tests": { "$ref": "#/$defs/Tests" }, "isBonus": { "description": "Points are bonus points.", "type": "boolean" }, "points": { "description": "Points, equally divided for each test.", "type": "integer", "minimum": 0, "maximum": 100 }, "preconditions": { "description": "If the precondition fails, no further tests are analyzed and 0 points are given. It contains an array of preconditions (test, optionally failure (text) and suite). Order of preconditions matter (short circuit evaluation)!", "type": "array", "items": { "$ref": "#/$defs/Precondition" } }, "image": { "description": "Deprecated, use images instead. Optional, if specified, the image is shown in the PDF if present.", "$ref": "#/$defs/Image" }, "images": { "description": "Optional, if specified, the images are shown in the PDF if present.", "type": "array", "items": { "$ref": "#/$defs/Image" } }, "text": { "description": "If text is omitted, the name of the first test is used.", "type": "string" } } }, "Precondition": { "description": "A precondition is just a normal test. If that test failed, the grading is skipped and zero points are registered.", "type": "object", "additionalProperties": false, "required": [ "test" ], "properties": { "suite": { "description": "Test suite, this may be omitted if task define a test suite.", "type": "string" }, "test": { "description": "Name of the test (simple name inside the test suite). If the test fails, the actual tests are ignored.", "type": "string" }, "failure": { "description": "Text to be displayed if test, in the case the precondition, failed", "type": "string" } } }, "Penalty": { "description": "Similar to grading, except that points must be negative. In contrast to a precondition, the grading is still done. If a penalty test is not found, it is treated as failed (error) -- however, it is has preconditions, it is skipped if the precondition fails.", "type": "object", "additionalProperties": false, "required": [ "tests", "points" ], "properties": { "suite": { "description": "Test suite, this may be omitted if the test suite is defined in the task.", "type": "string" }, "tests": { "$ref": "#/$defs/Tests" }, "points": { "description": "Points, equally divided for each test. The points are expected to be negative.", "type": "integer", "minimum": -100, "maximum": 0 }, "preconditions": { "description": "If the precondition fails, no further tests are analyzed. If it fails, 0 points are subtracted. Preconditions in penalties are used to avoid duplicate subtraction of points", "type": "array", "items": { "$ref": "#/$defs/Precondition" } }, "text": { "description": "If text is omitted, the name of the first test is used.", "type": "string" } } }, "Tests": { "description": "The actual tests", "type": "array", "items": { "description": "Name of the test (simple name inside the test suite).", "type": "string" } }, "Image": { "description": "The image is shown in the PDF if present. Images are to be generated in /reports/_${SUBMISSIONID}_img", "type": "object", "additionalProperties": false, "required": [ "src", "text" ], "properties": { "src": { "description": "The simple name, the image is expected to be in /reports/_${SUBMISSIONID}_img", "type": "string" }, "text": { "description": "The title of the image.", "type": "string" }, "missing": { "description": "Text to be displayed if the image is missing.", "type": "string" }, "width": { "description": "Width of the image as in latex, e.g. 10cm (which is the default)", "type": "string" } } } } }