@testresult/playwright-reporter/testresult-reporter.js

A Playwright reporter that sends test results to TestResult.co for analysis and tracking

const axios = require("axios");
const crypto = require("crypto");

class TestResultReporter {
  constructor(config = {}) {
    const apiUrl = process.env.API_URL;
    this.apiUrl =
      apiUrl && apiUrl.trim() !== ""
        ? apiUrl.trim()
        : "https://app.testresult.co/api/v1";

    // Remove any trailing slashes to ensure consistent URL format
    this.apiUrl = this.apiUrl.replace(/\/+$/, "");

    this.apiKey = process.env.API_KEY;
    this.testSuiteUuid = process.env.TEST_SUITE_UUID;
    this.runId = process.env.TEST_RUN_ID || crypto.randomUUID();
    this.debugReport =
      (process.env.DEBUG_REPORT || "0") === "1" ||
      (process.env.DEBUG_REPORT === undefined && config.debugReport) ||
      false;
    this.muteConsole =
      process.env.MUTE_CONSOLE === "1" || config.muteConsole || false;

    this.fullReport = {
      suites: [],
      errors: [],
      reportSlowTests: {},
      shardInfo: null,
    };
    this.projects = new Map();
    this.startTime = null;
    this.endTime = null;
    this.lifecycleTimings = {
      globalSetup: null,
      globalTeardown: null,
      beforeAll: {},
      afterAll: {},
      beforeEach: {},
      afterEach: {},
    };
    this.stepTimings = new Map();
    this.shardInfo = null;
  }

  // Runs when the test suite starts
  onBegin(config, suite) {
    this.startTime = new Date();
    this.fullReport = {
      suites: [],
      errors: [],
      reportSlowTests: {},
      shardInfo: null,
    };
    for (const project of config.projects) {
      if (!project.name) {
        throw new Error("Project name is undefined");
      }
      this.projects.set(project.name, project);
    }

    // Extract reportSlowTests values from config
    if ("reportSlowTests" in config) {
      this.fullReport.reportSlowTests = {
        threshold: config.reportSlowTests.threshold,
        max: config.reportSlowTests.max,
      };
    }

    // Capture shard information if it exists
    if ("shard" in config) {
      const shard = config.shard;
      if (
        shard &&
        typeof shard.current === "number" &&
        typeof shard.total === "number"
      ) {
        this.shardInfo = {
          current: shard.current,
          total: shard.total,
          startTime: this.startTime.toISOString(),
          runId: this.runId,
        };
        this.fullReport.shardInfo = this.shardInfo;
      }
    }
  }

  onTestBegin(test) {
    const testId = this.generateUniqueTestId(test);
    this.lifecycleTimings.beforeEach[testId] = { startTime: new Date() };
  }

  generateUniqueTestId(test) {
    const project = test.parent?.project();
    if (!project || !project.name) {
      throw new Error("Test project or project name is undefined");
    }
    if (!this.testSuiteUuid) {
      throw new Error("Test suite UUID is undefined");
    }
    const projectName = project.name;
    const testPath = `${test.location.file}`;
    const testTitle = test.title;
    const uniqueString = `${this.testSuiteUuid}|${projectName}|${testPath}|${testTitle}`;

    // Generate a SHA-256 hash of the unique string
    return crypto.createHash("sha256").update(uniqueString).digest("hex");
  }

  onTestEnd(test, result) {
    const project = test.parent?.project();
    if (!project || !project.name) {
      throw new Error("Test project or project name is undefined");
    }
    const projectName = project.name;
    const status = result.status.toUpperCase();

    if (!this.muteConsole) {
      const coloredStatus =
        status === "FAILED"
          ? `\x1b[31m${status}\x1b[0m`
          : status === "PASSED"
          ? `\x1b[32m${status}\x1b[0m`
          : `\x1b[33m${status}\x1b[0m`;

      console.log(`[${projectName}] ${test.title} - ${coloredStatus}`);
    }

    const uniqueTestId = this.generateUniqueTestId(test);
    const suite = this.findOrCreateSuite(test.parent);

    suite.specs.push({
      id: uniqueTestId,
      title: test.title,
      ok: result.status === "passed",
      tags: test.tags,
      file: test.location.file,
      line: test.location.line,
      column: test.location.column,
      project: projectName,
      tests: [
        {
          id: uniqueTestId,
          timeout: test.timeout,
          annotations: test.annotations,
          expectedStatus: test.expectedStatus,
          projectName: projectName,
          results: [this.serializeTestResult(result)],
        },
      ],
    });

    if (this.lifecycleTimings.beforeEach[uniqueTestId]?.startTime) {
      this.lifecycleTimings.beforeEach[uniqueTestId].endTime = new Date();
      this.lifecycleTimings.beforeEach[uniqueTestId].duration =
        this.lifecycleTimings.beforeEach[uniqueTestId].endTime.getTime() -
        this.lifecycleTimings.beforeEach[uniqueTestId].startTime.getTime();
    }

    const lastSpec = suite.specs[suite.specs.length - 1];
    lastSpec.lifecycleTimings = {
      beforeEach: this.lifecycleTimings.beforeEach[uniqueTestId]?.duration,
      afterEach: this.lifecycleTimings.afterEach[uniqueTestId]?.duration,
      beforeAll: this.lifecycleTimings.beforeAll[test.parent.title]?.duration,
      afterAll: this.lifecycleTimings.afterAll[test.parent.title]?.duration,
    };
  }

  onStepBegin(test, result, step) {
    const stepId = `${this.generateUniqueTestId(test)}-${step.title}`;
    this.stepTimings.set(stepId, { startTime: new Date() });
  }

  onStepEnd(test, result, step) {
    const stepId = `${this.generateUniqueTestId(test)}-${step.title}`;
    const timing = this.stepTimings.get(stepId);
    if (timing) {
      timing.endTime = new Date();
      timing.duration = timing.endTime.getTime() - timing.startTime.getTime();
      step.duration = timing.duration;
    }
  }

  onError(error) {
    this.fullReport.errors.push(error);
  }

  async onEnd(result) {
    this.endTime = new Date();
    this.fullReport.result = result;
    this.fullReport.startTime = this.startTime.toISOString();
    this.fullReport.endTime = this.endTime.toISOString();
    this.fullReport.duration =
      this.endTime.getTime() - this.startTime.getTime();

    // Update shard timing information
    // * Future feature *
    if (this.shardInfo) {
      this.shardInfo.endTime = this.endTime.toISOString();
      this.shardInfo.duration =
        this.endTime.getTime() - this.startTime.getTime();
      this.fullReport.shardInfo = this.shardInfo;
    }

    // Write debug output if debugReport is enabled
    if (this.debugReport) {
      const debugDir = "playwright-debug";
      const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
      const shardSuffix = this.shardInfo
        ? `-shard-${this.shardInfo.current}`
        : "";
      const filename = `test-results-${timestamp}${shardSuffix}.json`;

      try {
        // Create debug directory if it doesn't exist
        if (!require("fs").existsSync(debugDir)) {
          require("fs").mkdirSync(debugDir, { recursive: true });
        }

        // Write the full report to a JSON file
        require("fs").writeFileSync(
          `${debugDir}/${filename}`,
          JSON.stringify(
            {
              url: `${this.apiUrl}/test_suites/${this.testSuiteUuid}/test_runs.json`,
              payload: {
                test_run: {
                  results: this.fullReport,
                  started_at: this.startTime.toISOString(),
                  ended_at: this.endTime.toISOString(),
                  duration: this.fullReport.duration,
                  lifecycle_timings: {
                    globalSetup: this.lifecycleTimings.globalSetup?.duration,
                    globalTeardown:
                      this.lifecycleTimings.globalTeardown?.duration,
                  },
                  shard_info: this.shardInfo,
                },
              },
            },
            null,
            2
          )
        );
        console.log(`Debug output written to ${debugDir}/${filename}`);
      } catch (error) {
        console.error("Error writing debug output:", error);
      }
    }
    if (!this.apiUrl || !this.testSuiteUuid) {
      console.error(
        "\x1b[31mError: apiUrl or testSuiteUuid is undefined\x1b[0m"
      );
      return;
    }

    // Modify the URL to include shard information if present
    // * Future feature *
    let url = `${this.apiUrl}/test_suites/${this.testSuiteUuid}/test_runs.json`;
    if (this.shardInfo) {
      url += `?shard=${this.shardInfo.current}&total_shards=${this.shardInfo.total}&run_id=${this.runId}`;
    }

    if (!this.muteConsole) {
      console.log("\n📊 Test Run Summary:");
      console.log(`Duration: ${(this.fullReport.duration / 1000).toFixed(2)}s`);
      console.log(`Total Tests: ${this.getTotalTests()}`);
      console.log(`Status: ${result.status.toUpperCase()}`);

      if (this.shardInfo) {
        console.log(`Shard: ${this.shardInfo.current}/${this.shardInfo.total}`);
      }

      if (this.debugReport) {
        console.log("\nSending results to TestResult.co:");
        console.log(`URL: ${url}`);
      }
    }

    const payload = {
      test_run: {
        results: this.fullReport,
        started_at: this.startTime.toISOString(),
        ended_at: this.endTime.toISOString(),
        duration: this.fullReport.duration,
        lifecycle_timings: {
          globalSetup: this.lifecycleTimings.globalSetup?.duration,
          globalTeardown: this.lifecycleTimings.globalTeardown?.duration,
        },
        shard_info: this.shardInfo,
      },
    };

    try {
      if (this.debugReport && !this.muteConsole) {
        console.log("\nPayload:", JSON.stringify(payload, null, 2));
      }

      const response = await axios.post(url, payload, {
        headers: {
          "Content-Type": "application/json",
          "X-API-Key": this.apiKey,
          "User-Agent": "TestResult-Reporter/1.0.0 (Playwright)",
        },
        timeout: 60000,
      });

      if (response.data.status === "processing") {
        if (!this.muteConsole) {
          console.log("\n⏳ Processing results...");
        }
        await this.pollForCompletion(response.data.id);
      }

      if (!this.muteConsole) {
        console.log("\n✅ Test results uploaded successfully");
      }
    } catch (error) {
      if (axios.isAxiosError(error)) {
        console.error("\n❌ Error sending test results:");
        if (error.code === "ECONNREFUSED") {
          console.error(
            `Connection refused - Unable to connect to ${this.apiUrl}`
          );
        } else {
          if (error.response) {
            console.error(`Status: ${error.response.status}`);
            console.error(
              `Message: ${error.response.data.message || error.message}`
            );
          } else {
            console.error(`Error: ${error.message}`);
          }
        }
        if (this.debugReport) {
          console.error("\nFull error:", error);
        }
      } else if (error instanceof Error) {
        console.error("\n❌ Error:", error.message);
      }
    }
  }

  getTotalTests() {
    return this.fullReport.suites.reduce((total, suite) => {
      return total + suite.specs.length;
    }, 0);
  }

  async pollForCompletion(testRunId) {
    const maxAttempts = 30;
    let attempts = 0;
    const startTime = Date.now();

    while (attempts < maxAttempts) {
      const response = await axios.get(
        `${this.apiUrl}/test_runs/${testRunId}/status`,
        {
          headers: {
            "X-API-Key": this.apiKey,
            "User-Agent": "TestResult-Reporter/1.0.0 (Playwright)",
          },
        }
      );

      if (response.data.status === "errored") {
        console.error("\n❌ Test run processing failed");
        process.exit(1);
      }

      if (["passed", "failed"].includes(response.data.status)) {
        if (!this.muteConsole) {
          const duration = ((Date.now() - startTime) / 1000).toFixed(1);
          console.log(`✅ Processing completed in ${duration}s`);
        }
        return response.data;
      }

      attempts++;
      if (!this.muteConsole && attempts % 3 === 0) {
        process.stdout.write(".");
      }
      await new Promise((resolve) => setTimeout(resolve, 5000));
    }

    throw new Error(
      "Timeout waiting for test run processing. Check TestResult.co for results."
    );
  }

  findOrCreateSuite(suite) {
    if (!suite) return { title: "", specs: [] };
    let found = this.fullReport.suites.find((s) => s.title === suite.title);
    if (!found) {
      found = { title: suite.title, specs: [] };
      this.fullReport.suites.push(found);
    }
    return found;
  }

  serializeTestResult(result) {
    return {
      duration: result.duration,
      status: result.status,
      error: result.error
        ? {
            message: result.error.message || "Unknown error",
            stack: result.error.stack,
          }
        : undefined,
      stdout: result.stdout.map((output) => output.toString()),
      stderr: result.stderr.map((output) => output.toString()),
      retry: result.retry,
      steps: result.steps.map((step) => this.serializeTestStep(step)),
      attachments: result.attachments,
    };
  }

  serializeTestStep(step) {
    return {
      title: step.title,
      category: step.category,
      duration: step.duration,
      error: step.error
        ? {
            message: step.error.message || "Unknown error",
            stack: step.error.stack,
          }
        : undefined,
      steps: step.steps.map((s) => this.serializeTestStep(s)),
    };
  }

  onGlobalSetup() {
    this.lifecycleTimings.globalSetup = { startTime: new Date() };
  }

  onGlobalTeardown() {
    this.lifecycleTimings.globalTeardown = { startTime: new Date() };
    this.lifecycleTimings.globalTeardown.endTime = new Date();
    this.lifecycleTimings.globalTeardown.duration =
      this.lifecycleTimings.globalTeardown.endTime.getTime() -
      this.lifecycleTimings.globalTeardown.startTime.getTime();
  }

  onBeforeAll(suite) {
    this.lifecycleTimings.beforeAll[suite.title] = { startTime: new Date() };
  }

  onAfterAll(suite) {
    if (this.lifecycleTimings.beforeAll[suite.title]?.startTime) {
      this.lifecycleTimings.afterAll[suite.title] = {
        startTime: new Date(),
        endTime: new Date(),
      };
      this.lifecycleTimings.afterAll[suite.title].duration =
        this.lifecycleTimings.afterAll[suite.title].endTime.getTime() -
        this.lifecycleTimings.afterAll[suite.title].startTime.getTime();
    }
  }
}

module.exports = TestResultReporter;