grading/dist/grade/grading.ts

Grading of student submissions, in particular programming tests.

import { error, JSONSrcMap, log, verb, warn } from '../cli/cliUtil';
import { Testcase, Testsuite, Testsuites } from "../junit";
import { Submitter } from "../cli/submitter";
import { GradingSchema, gradingText, preconditionText } from "./gradingschema";
import { loadCoverageReport, loadUnitReport } from './reportLoader';
import { gradeFromPoints, GradingResult, ManualCorrection, Results, StudentResult, TaskResult, TestStatus } from "./results";
import { CorrectionSchema, StudentCorrection } from './correctionschema';
import { didYouMeanTest } from './didYouMean';
import { getPatchFileName } from '../gitCommands';
import { fileExists } from '../fsUtil';
import { sep } from 'path';


export async function doGrading({ gradingSchema, manualCorrection,
    submitters, reportsDir, startMarker, endMarker,
    patchFolder }:
    {
        gradingSchema: GradingSchema, manualCorrection: CorrectionSchema | undefined,
        submitters: Submitter[], reportsDir: string, startMarker: string, endMarker: string,
        patchFolder: string
    }
): Promise<Results> {

    log("Running Grading");
    if (!JSONSrcMap.hasMap(gradingSchema)) {
        warn("No source map for grading schema available.");
    }

    if (!validate(gradingSchema)) {
        program.error(`Grading schema inconsistency: points to not add up.`)
    }

    const results = new Results(gradingSchema.course, gradingSchema.term, gradingSchema.exam,
        gradingSchema.points,
        gradingSchema.minCoverageStatements, gradingSchema.penaltiesCoverageMax, gradingSchema.extraCoverageMax);

    const appliedCorrections: StudentCorrection[] = []

    // for each submitter
    for (const submitter of submitters) {
        if (!submitter.submissionId) {
            throw new Error("Submission ID is missing for Submitter " + submitter.name);
        }
        const studentResult = new StudentResult(submitter.submissionId, submitter.name);
        const patchFileName = getPatchFileName(patchFolder, submitter.name, submitter.submissionId);
        studentResult.patched = await fileExists(patchFileName, 'patch file');

        const studentCorrection: StudentCorrection | undefined = manualCorrection?.corrections.find(c => c.submissionID == studentResult.submissionId); // === doest not work here, id may be a number
        const absolutePoints = studentCorrection?.general?.find(c => c.absolute)?.points ?? -1;

        try {

            results.studentResults.push(studentResult);

            // if (bewertungstabelle.at("Status", row).startsWith("Keine Abgabe")) {
            //     studentResult.noSubmission = true;
            // } else { // Abgabe erfolgt

            // Lese junit.check, junit.student
            // bei Error wird null zurückgegeben:
            const unitCheck = await loadUnitReport(reportsDir, submitter.submissionId, "check");
            const unitStudent =
                gradingSchema.minCoverageStatements
                    ? await loadUnitReport(reportsDir, submitter.submissionId, "student")
                    : null;

            studentResult.userTests = unitStudent ? true : false;
            if (unitStudent) {
                studentResult.studentReportFound = true;
                studentResult.userTestsCount = unitStudent.tests;
                studentResult.userTestsSuccess = unitStudent.tests - (unitStudent.errors + unitStudent.failures + unitStudent.skipped);
                studentResult.userTestsFailureOrError = unitStudent.errors + unitStudent.failures;
            } else {
                studentResult.studentReportFound = false;
            }

            if (!unitCheck) {
                studentResult.checkReportFound = false;

                warn(`${submitter.name}: Warning, no check report found!`);
                studentResult.skipMessage = `Es konnte keine Bewertung durchgeführt werden. 
                        Entweder wurde nichts abgegeben oder die Projektstruktur entspricht nicht den Vorgaben. 

                        Unter Umständen hat ein Test auch eine Endlosschleife. In diesem Fall kann ebenfalls keine automatische Bewertung vorgenommen werden.`

                if (!unitStudent) {
                    if (gradingSchema.minCoverageStatements) {
                        studentResult.errorDetails += "Es konnte auch kein Test-Report der eigenen Tests gefunden werden. ";
                    }
                } else {
                    if (unitStudent.tests == 0) {
                        studentResult.errorDetails += "Es wurden keine eigenen Tests gefunden."
                    } else {
                        studentResult.errorDetails += "Es wurden " + unitStudent.tests + " gefunden, von denen " + (unitStudent.errors + unitStudent.failures) + " fehlerhaft waren."
                    }
                }
            } else {
                studentResult.checkReportFound = true;
                if (gradingSchema.minCoverageStatements) { // neither undefined nor 0
                    try {
                        const coverageStudentCheck = await loadCoverageReport(reportsDir, submitter.submissionId, "student");
                        studentResult.coverageResult.stmtCoverage = 0;
                        if (!coverageStudentCheck) {
                            studentResult.coverageResult.skipMessage = `Es konnte kein Coverage-Report erstellt werden.
                                 Es wird daher eine Coverage von 0\\% angenommen.`
                        } else {
                            const statements = coverageStudentCheck.project.metrics.statements;
                            const coveredStatements = coverageStudentCheck.project.metrics.coveredstatements;
                            if (statements > 0) {
                                let stmtCoverage = Math.round(100 * coveredStatements / statements);
                                if (stmtCoverage < 0) {
                                    warn(`${submitter.name}: Coverage less then 0, set to 0.`)
                                    stmtCoverage = 0;
                                } else if (stmtCoverage > 100) {
                                    warn(`${submitter.name}: Coverage greater than 100, set to 0.`)
                                    stmtCoverage = 0;

                                }
                                studentResult.coverageResult.stmtCoverage = stmtCoverage;
                            } else {
                                studentResult.coverageResult.skipMessage = "Es wurden keine Statements gefunden. Es wird daher eine Coverage von 0\\% angenommen."
                            }
                        }

                    } catch (err) {
                        studentResult.coverageResult.skipMessage = "Beim Analysieren der Coverage ist ein Fehler aufgetreten. Es wird daher eine Coverage von 0\\% angenommen."
                    }
                }

                verb(`${gradingSchema.penalties ? "Penalities defined" : "No penalities defined"}`)


                for (const penalty of gradingSchema.penalties ?? []) {
                    const gradingPenalty = new GradingResult(gradingText(penalty), penalty.points, false);
                    studentResult.addPenalty(gradingPenalty);

                    if (!penalty.suite) {
                        warn("Penalty does not define a test suite. Top level penalties require their own test suite.");
                    }
                    let testSuiteOfPenalty = penalty.suite ? findSuite(unitCheck, penalty.suite) : undefined;
                    if (!testSuiteOfPenalty) {
                        verb(`${submitter.name}: Warning: Top-Level Penalty Testsuite ${penalty.suite} not found for grading.`);
                        // gradingPenalty.skipMessage = `Die Aufgabe ${task.name} konnte nicht vollständig überprüft werden. Die Überprüfung zu "${gradingPenalty.text}" konnte nicht durchgeführt werden.\n`;
                    } else {
                        verb(`Top-Level Penalty: ${gradingPenalty.text}, suite: ${penalty.suite}, found`);
                    }
                    ///////////////////////
                    let preConditionFailed = false;
                    for (let pre of penalty.preconditions ?? []) {
                        let preTest = testSuiteOfPenalty ? findTestCase(testSuiteOfPenalty, pre.test) : undefined;
                        if (testNotSuccessful(preTest)) {
                            preConditionFailed = true;
                            break;
                        }
                    }
                    ///////////////////////
                    if (!preConditionFailed) { // kein Abzug, wenn precondition fehlschlägt)
                        for (let test of penalty.tests) {
                            let status: TestStatus = "success";
                            let message: string | undefined = undefined;
                            try {
                                let bewTest = testSuiteOfPenalty ? findTestCase(testSuiteOfPenalty, test) : undefined;
                                if (!bewTest) {
                                    warn(`${JSONSrcMap.getLocation(gradingSchema, test)}${submitter.name}: Warning, Penalty test "${test}" of grading not found.${didYouMeanTest(testSuiteOfPenalty, test)}`);
                                    status = "error"; // "notFound";
                                } else {
                                    [status, message] = getTestCaseStatus(bewTest, startMarker, endMarker);
                                }
                            } catch (err) {
                                status = "notFound";
                            }
                            gradingPenalty.addTestResult(test, status, message);
                        }
                    }
                }


                for (let task of gradingSchema.tasks) {
                    const taskResult = new TaskResult(task.name, task.points);
                    studentResult.taskResults.push(taskResult);

                    if (task.image) {
                        taskResult.images.push(task.image);
                    }
                    if (task.images) {
                        taskResult.images.push(...task.images);
                    }

                    try {

                        let testSuiteOfTask: Testsuite | undefined = undefined;
                        let skipMessage: string = "";

                        if (task.suite) {
                            testSuiteOfTask = findSuite(unitCheck, task.suite);

                            if (!testSuiteOfTask) {
                                warn((`${submitter.name}: Warning: Testsuite ${task.suite} not found for task ${task.name}.`));
                                skipMessage = `Die Aufgabe ${task.name} kann nicht vollständig überprüft werden.\n`;
                                if (task.skipHint) {
                                    skipMessage += task.skipHint;
                                } else {
                                    skipMessage += `Vermutlich wurde eine Datei oder Deklaration falsch benannt, liegt nicht vor oder enthält kann nicht übersetzt werden.`;
                                }
                            }
                        }

                        if (task.preconditions) { // test preconditions, even if task test suite was not found. This may add information!
                            for (let pre of task.preconditions) {

                                const testSuiteOfPrecondition = (pre.suite) ? findSuite(unitCheck, pre.suite) : testSuiteOfTask;
                                if (!testSuiteOfPrecondition) {
                                    warn((`${submitter.name}: Warning: Testsuite ${pre.suite} not found for precondition.`));
                                    skipMessage += `\nDie Vorbedingung konnte nicht überprüft werden. ${pre.failure}\n`;
                                } else { // if (testSuiteOfPrecondition) {
                                    let preTest = findTestCase(testSuiteOfPrecondition, pre.test);
                                    if (!preTest) {
                                        warn(`${submitter.name}: Warning, precondition test "${pre.test}" not found.${didYouMeanTest(testSuiteOfPrecondition, pre.test)}`);
                                        skipMessage += `\n${preconditionText(pre)} Unter Umständen wirft die Funktionen einen Fehler oder eine Deklaration fehlt.`;
                                        taskResult.precondition = "notFound";
                                        break;
                                    } else {
                                        if (testNotSuccessful(preTest)) {
                                            const message = extractMessage(preTest.error ? preTest.error : preTest.failure, startMarker, endMarker);
                                            skipMessage += `\n${preconditionText(pre)} ${message ? message : ""}`;
                                            taskResult.precondition = "failed";
                                            break;
                                        } else {
                                            taskResult.precondition = "success";
                                        }
                                    }
                                }
                            }
                        }

                        
                        taskResult.message = findTestSuiteMessage(testSuiteOfTask, startMarker, endMarker);

                        // console.log(`${submitter.name}: Task: ${task.name}, Precon: ${taskResult.precondition}, Skipmsg: ${skipMessage}`);

                        if (skipMessage) {
                            taskResult.skipMessage = skipMessage;
                        } else { // perform grading
                            for (let grading of task.grading) {
                                const gradingResult = new GradingResult(gradingText(grading), grading.points, grading.isBonus);
                                taskResult.addGradingResult(gradingResult);

                                if (grading.image) {
                                    gradingResult.images.push(grading.image);
                                }
                                if (grading.images) {
                                    gradingResult.images.push(...grading.images);
                                }

                                const testSuiteOfGrading = (grading.suite) ? findSuite(unitCheck, grading.suite) : testSuiteOfTask;
                                if (!testSuiteOfGrading) {
                                    warn((`${submitter.name}: Warning: Testsuite ${grading.suite} not found for grading.`));
                                    gradingResult.skipMessage = `Die Aufgabe ${task.name} konnte nicht vollständig überprüft werden. Die Bewertung zu "${gradingResult.text}" konnte nicht durchgeführt werden.\n`;
                                } else {
                                    if (grading.preconditions) { // test preconditions, even if main test suite was not found. This may add information!
                                        for (let pre of grading.preconditions) {
                                            const testSuiteOfPrecondition = pre.suite ? findSuite(unitCheck, pre.suite) : testSuiteOfGrading;
                                            if (!testSuiteOfPrecondition) {
                                                warn((`${submitter.name}: Warning: Testsuite ${pre.suite} not found for precondition.`));
                                                gradingResult.skipMessage = `Die Bewertung zu "${gradingResult.text}" konnte nicht durchgeführt werden.\nVorbedingung konnte nicht überprüft werden: ${pre.failure || pre.test}\n`;
                                            } else { // } if (testSuiteOfPrecondition) {
                                                let preTest = findTestCase(testSuiteOfGrading!, pre.test);
                                                if (!preTest) {
                                                    warn(`${submitter.name}: Warning: Precondition test "${pre.test}" not found.`);
                                                    gradingResult.skipMessage = `Die grundlende Funktion von "${gradingResult.text}" konnte nicht überprüft werden.\n` +
                                                        `Unter Umständen wirft die Funktion einen Fehler.`;
                                                    gradingResult.precondition = "notFound";
                                                    break;
                                                } else {
                                                    if (testNotSuccessful(preTest)) { // TODO: extract message in error/failure
                                                        gradingResult.skipMessage = preconditionText(pre);
                                                        gradingResult.precondition = "failed";
                                                        break;
                                                    } else {
                                                        gradingResult.precondition = "success";
                                                    }
                                                }
                                            }
                                        }
                                    }
                                    gradingResult.message = findTestSuiteMessage(testSuiteOfGrading, startMarker, endMarker);
                                    ///////////////////////
                                    if (!gradingResult.skipMessage) { // not skipped due to failed precondition
                                        for (let test of grading.tests) {
                                            let status: TestStatus = "success";
                                            let message: string | undefined = undefined
                                            try {

                                                let testCase = findTestCase(testSuiteOfGrading, test);
                                                if (!testCase) {
                                                    warn(`${JSONSrcMap.getLocation(gradingSchema, test)}${submitter.name}: Grading test "${test}" of task "${task.name}" not found.${didYouMeanTest(testSuiteOfGrading, test)}`);
                                                    status = "notFound";
                                                } else {
                                                    [status, message] = getTestCaseStatus(testCase, startMarker, endMarker);
                                                }
                                            } catch (err) {
                                                status = "notFound";
                                            }
                                            gradingResult.addTestResult(test, status, message);
                                        }
                                        

                                    }
                                }
                            } // end of for (let grading of task.grading)

                            //////////////////
                            for (let penalty of task.penalties ?? []) {
                                const gradingPenalty = new GradingResult(gradingText(penalty), penalty.points, false);
                                taskResult.addPenalty(gradingPenalty);

                                let testSuiteOfPenalty = (penalty.suite) ? findSuite(unitCheck, penalty.suite) : testSuiteOfTask;
                                if (!testSuiteOfPenalty) {
                                    // warn((`${submitter.name}: Warning: Testsuite ${penalty.suite} not found for grading.`));
                                    // gradingPenalty.skipMessage = `Die Aufgabe ${task.name} konnte nicht vollständig überprüft werden. Die Überprüfung zu "${gradingPenalty.text}" konnte nicht durchgeführt werden.\n`;
                                }
                                ///////////////////////
                                let preConditionFailed = false;
                                for (let pre of penalty.preconditions ?? []) {
                                    let preTest = testSuiteOfPenalty ? findTestCase(testSuiteOfPenalty, pre.test) : undefined;
                                    if (testNotSuccessful(preTest)) {
                                        preConditionFailed = true;
                                        break;
                                    }
                                }
                                ///////////////////////
                                if (!preConditionFailed) { // kein Abzug, wenn precondition fehlschlägt)
                                    for (let test of penalty.tests) {
                                        let status: TestStatus = "success";
                                        let message: string | undefined = undefined;
                                        try {
                                            let bewTest = testSuiteOfPenalty ? findTestCase(testSuiteOfPenalty, test) : undefined;
                                            if (!bewTest) {
                                                warn(`${JSONSrcMap.getLocation(gradingSchema, test)}${submitter.name}: Warning, Penalty test "${test}" of task "${task.name}" not found.${didYouMeanTest(testSuiteOfPenalty, test)}`);
                                                status = "error"; // "notFound";
                                            } else {
                                                [status, message] = getTestCaseStatus(bewTest, startMarker, endMarker);
                                            }
                                        } catch (err) {
                                            status = "notFound";
                                        }
                                        gradingPenalty.addTestResult(test, status, message);
                                    }
                                }

                            }
                        }
                    } catch (err) {
                        error(err);
                        taskResult.skipMessage = "Bei der Bewertung der Aufgabe ist ein Fehler aufgetreten.";
                    }
                } // end for (let task of gradingSchema.tasks)
            }


            // perform manual corrections
            if (studentCorrection) {
                appliedCorrections.push(studentCorrection);
                if (studentResult.userName !== studentCorrection.userName) {
                    error(`Student with submission ID "${studentCorrection.submissionID}" has name "${studentResult.userName}" but correction file has name "${studentCorrection.userName}".`);
                } else {
                    verb(`Applying manual corrections for student "${studentResult.submissionId}" (${studentResult.userName})`);
                    if (studentCorrection.general) {
                        studentCorrection.general.forEach(generalCorrection => {
                            studentResult.manualCorrection.push(new ManualCorrection(generalCorrection.points, generalCorrection.reason, generalCorrection.absolute));
                        });
                    }
                    if (studentCorrection.tasks) {
                        studentCorrection.tasks.forEach(taskCorrection => {
                            const taskResult = studentResult.taskResults.find(taskResult => taskResult.name == taskCorrection.name);
                            if (!taskResult) {
                                error(`Task "${taskCorrection.name}" not found for student with submission ID "${studentCorrection.submissionID}", cannot apply manual correction. Known tasks: ${studentResult.taskResults.map(taskResult => `'${taskResult.name}'`).join(", ")}`);
                            } else {
                                taskResult.manualCorrections.push(new ManualCorrection(taskCorrection.points, taskCorrection.reason, false));
                            }
                        });
                    }
                }
            }


            try {
                studentResult.computeAndSetRawPunkte();
                // set, just in case coveragePunkte throws an exception:
                studentResult.points = studentResult.pointsRaw;
                if (studentResult.points > results.maxPoints) {
                    studentResult.points = results.maxPoints
                }

                // now compute coverage punkte
                studentResult.pointsCoverage = results.coveragePunkte(studentResult.pointsRaw, studentResult.coverageResult.stmtCoverage);
                // and now adjust with coverage:
                studentResult.points = studentResult.pointsRaw + studentResult.pointsCoverage;
                if (studentResult.points > results.maxPoints) {
                    studentResult.points = results.maxPoints
                }
                if (absolutePoints >= 0) {
                    studentResult.points = absolutePoints;
                    studentResult.pointsRaw = absolutePoints;
                    studentResult.pointsCoverage = 0;
                    studentResult.absolutePoints = true;
                    studentResult.addComment("Die Punkte wurden manuell gesetzt. Die Punkte in der Aufstellung sind nur zur Information angegeben und spielen keine Rolle.")
                }

                studentResult.grade = gradeFromPoints(studentResult.points, results.gradingTable);
                studentResult.gradingDone = true;


            } catch (err) {
                error(`${submitter.name}: Error assigning points: ${err}`);
            }

        } catch (err) {
            error(err);
            studentResult.skipMessage = `Bei der Bewertung der Abgabe ist ein Fehler aufgetreten.`
        }
    }

    if (manualCorrection) { // correctionSchema.corrections
        results.appliedCorrections = appliedCorrections.length;
        const notFoundCorrections = manualCorrection.corrections.filter(c => !appliedCorrections.includes(c));
        if (notFoundCorrections.length > 0) {
            warn(`The following corrections were not applied:`);
            notFoundCorrections.forEach(c => {
                warn(`- ${c.submissionID} (${c.userName})`);
            });
        }
    }

    return results;
}



export function validate(gradingSchema: GradingSchema): boolean {
    let valid = true;

    if (gradingSchema.points === undefined) {
        error(`Error: Exam does not declare grading points.`);
        valid = false;
    }

    let sumPointsOfTasks = gradingSchema.tasks.reduce((sum, task) => sum += task.points, 0);
    if (sumPointsOfTasks != gradingSchema.points) {
        if (gradingSchema.points !== undefined) {
            error(`Error: Points declared in exam (${gradingSchema.points}) differ from sum of points of tasks (${sumPointsOfTasks}).`);
        } else {
            error(`Error: Sum of points of tasks is ${sumPointsOfTasks}.`);
        }
        valid = false;
    }

    if (gradingSchema.penalties?.some(bew => bew.points > 0)) {
        error(`Error: Points of penalities (${gradingSchema.points}) of grading schema are positive, only allowed for grading.`)
        valid = false;
    }

    for (const task of gradingSchema.tasks) {
        if (task.grading.some(grading => grading.points <= 0)) {
            error(`Error: Points (${task.points}) of task "${task.name}" are negative or zero, only allowed for penalties.`)
            valid = false;
        }
        if (task.penalties?.some(bew => bew.points > 0)) {
            error(`Error: Points of penalities (${task.points}) of task "${task.name}" are positive, only allowed for grading.`)
            valid = false;
        }
        if (!task.manual) {
            if (task.grading.length == 0) {
                error(`Error: Task "${task.name}" is not manual, but has no grading settings.`)
                valid = false;
            } else {
                const sumGrading = task.grading.reduce((sum, grading) => {
                    if (!grading.isBonus) {
                        return sum + grading.points
                    } else {
                        return sum;
                    }
                }, 0);
                if (sumGrading !== task.points) {
                    error(`Error: Sum of points (${task.points}) of task "${task.name}" differ from declared total points of task (${sumGrading}).`)
                    valid = false;
                }
            }
        } else {
            if (task.grading.length > 0) {
                error(`Error: Task "${task.name}" is manual, but has grading settings.`)
                valid = false;
            }
        }
    }

    for (const task of gradingSchema.tasks) {
        let gradingSuites = 0;
        if (!task.manual) {
            task.grading.forEach((grading, index) => {
                if (!task.suite && !grading.suite) {
                    error(`Grading ${index + 1} of task ${task.name} has no test suite defined. It is required since test does not define a suite.`)
                }
                if (grading.suite) {
                    gradingSuites++;
                }
            });
            if (task.grading.length == gradingSuites && task.suite) {
                error(`All ${gradingSuites} grading settings of task ${task.name} define their own suite. Remove task suite as it has no effect.`);
            }
        }
    }

    return valid;
}

/**
 * Returns the test name with the given name or the first test that ends with the given name.
 * 
 * Note: Jest ignores suits created with describe, while in Vitest the full path is stored separeted by ">" (i.e. " &gt; ").
 * @param testSuite 
 * @param testName 
 * @returns 
 */
function findTestCase(testSuite: Testsuite, testName: string) {
    const exactMatch = testSuite.testcase.find(testcase => testcase.name === testName);
    if (exactMatch) {
        return exactMatch;
    }
    const testNameWithSpace = " " + testName;
    return testSuite.testcase.find(testcase => testcase.name.endsWith(testNameWithSpace));
}

/**
 * Returns the test suite with the given name or the first suite that ends with the given name.
 * 
 * Note: Jest just stores the last part of the suite name, while Vitest stores the full path.
 * @param unitCheck Ret
 * @param suiteName 
 * @returns 
 */
function findSuite(unitCheck: Testsuites, suiteName: string) {
    const exactMatch = unitCheck.testsuite.find(suite => suite?.name === suiteName);
    if (exactMatch) {
        return exactMatch;
    }
    const suiteNameWithSep = sep + suiteName;
    return unitCheck.testsuite.find(suite => suite?.name.endsWith(suiteNameWithSep));
}

/**
 * Exported for testing only
 */
export function extractMessage(output: string | undefined | { message: string }, startMarker: string, endMarker: string): string | undefined {
    if (!startMarker || !endMarker || !output) {
        return undefined;
    }
    if (typeof output === "object" && "message" in output) {
        output = output.message;
    }


    let start = output.indexOf(startMarker);
    if (start < 0) {
        return undefined;
    }
    start += startMarker.length;
    let end = output.indexOf(endMarker, start);
    if (end < 0) {
        return undefined;
    }
    if (end - start > 2000) {
        end = start + 2000;
    }
    const message = output.substring(start, end)
    return message;
}

function getTestCaseStatus(testCase: Testcase, startMarker: string, endMarker: string): [TestStatus, string | undefined] {
    let status: TestStatus;
    let message: string | undefined = undefined;

    if (testCase.error !== undefined) {
        status = "error";
        message = extractMessage(testCase.error, startMarker, endMarker);
    } else if (testCase.failure !== undefined) {
        status = "failure";
        message = extractMessage(testCase.failure, startMarker, endMarker);
    } else if (testCase.skipped !== undefined) {
        status = "skipped";
        message = extractMessage(testCase.skipped, startMarker, endMarker);
    } else {
        status = "success";
    }
    return [status, message];
}

function testNotSuccessful(testCase: Testcase | undefined): boolean {
    return !testCase
        || testCase.error !== undefined
        || testCase.failure !== undefined
        || testCase.skipped !== undefined;
}   

function findTestSuiteMessage(testSuite: Testsuite|undefined, startMarker: string, endMarker: string): string | null {
    if (!testSuite) {
        return null;
    }
    // <testcase classname="check/tests/fetch/GebietOhneRouter.check.ts" name="tests/fetch/GebietOhneRouter.check.ts" time="6.652957294">
    const testSuiteName = testSuite.name;
    const testSuiteCase = testSuite.testcase.find(testcase => testSuiteName.endsWith(testcase.name));
    if (testSuiteCase) {
        return getTestCaseStatus(testSuiteCase, startMarker, endMarker)[1] || null;
    }
    return null;
}