Reporting

Output formatters for test results.

Terminal Reporter

agentprobe.reporting.terminal

Terminal reporter using Rich for formatted output.

Produces colored tables, progress summaries, and status panels for agent test runs.

TerminalReporter

Reporter that outputs formatted results to the terminal.

Uses Rich for colored tables, panels, and progress indicators.

Attributes:

Name	Type	Description
console		Rich Console instance for output.

Source code in src/agentprobe/reporting/terminal.py

class TerminalReporter:
    """Reporter that outputs formatted results to the terminal.

    Uses Rich for colored tables, panels, and progress indicators.

    Attributes:
        console: Rich Console instance for output.
    """

    def __init__(self, console: Console | None = None) -> None:
        """Initialize the terminal reporter.

        Args:
            console: Rich Console to use. Creates a new one if None.
        """
        self._console = console or Console()

    @property
    def name(self) -> str:
        """Return the reporter name."""
        return "terminal"

    async def report(self, run: AgentRun) -> None:
        """Print a formatted test run report to the terminal.

        Args:
            run: The completed agent run.
        """
        self._print_header(run)
        self._print_results_table(run)
        self._print_summary(run)

    def _print_header(self, run: AgentRun) -> None:
        """Print the report header with agent name and status."""
        status_color = "green" if run.failed == 0 and run.errors == 0 else "red"
        self._console.print(
            Panel(
                f"[bold]{run.agent_name}[/bold] — {run.status.value}",
                title="AgentProbe Test Report",
                border_style=status_color,
            )
        )

    def _print_results_table(self, run: AgentRun) -> None:
        """Print a table of individual test results."""
        if not run.test_results:
            self._console.print("[dim]No test results.[/dim]")
            return

        table = Table(show_header=True, header_style="bold")
        table.add_column("Test", style="cyan", min_width=20)
        table.add_column("Status", justify="center", min_width=10)
        table.add_column("Score", justify="right", min_width=8)
        table.add_column("Duration", justify="right", min_width=10)
        table.add_column("Details", min_width=20)

        for result in run.test_results:
            color = _STATUS_COLORS.get(result.status, "white")
            details = ""
            if result.error_message:
                details = result.error_message[:50]
            elif result.eval_results:
                verdicts = [r.verdict.value for r in result.eval_results]
                details = ", ".join(verdicts)

            table.add_row(
                result.test_name,
                f"[{color}]{result.status.value}[/{color}]",
                f"{result.score:.2f}",
                f"{result.duration_ms}ms",
                details,
            )

        self._console.print(table)

    def _print_summary(self, run: AgentRun) -> None:
        """Print a summary panel with totals."""
        parts = [
            f"Total: {run.total_tests}",
            f"[green]Passed: {run.passed}[/green]",
            f"[red]Failed: {run.failed}[/red]",
        ]
        if run.errors > 0:
            parts.append(f"[red bold]Errors: {run.errors}[/red bold]")
        if run.skipped > 0:
            parts.append(f"[dim]Skipped: {run.skipped}[/dim]")
        parts.append(f"Duration: {run.duration_ms}ms")

        self._console.print()
        self._console.print(" | ".join(parts))

name property

Return the reporter name.

__init__(console=None)

Initialize the terminal reporter.

Parameters:

Name	Type	Description	Default
console	Console | None	Rich Console to use. Creates a new one if None.	None

Source code in src/agentprobe/reporting/terminal.py

def __init__(self, console: Console | None = None) -> None:
    """Initialize the terminal reporter.

    Args:
        console: Rich Console to use. Creates a new one if None.
    """
    self._console = console or Console()

report(run) async

Print a formatted test run report to the terminal.

Parameters:

Name	Type	Description	Default
run	AgentRun	The completed agent run.	required

Source code in src/agentprobe/reporting/terminal.py

async def report(self, run: AgentRun) -> None:
    """Print a formatted test run report to the terminal.

    Args:
        run: The completed agent run.
    """
    self._print_header(run)
    self._print_results_table(run)
    self._print_summary(run)

HTML Reporter

agentprobe.reporting.html

HTML reporter for test results.

Generates a standalone HTML file with embedded CSS for viewing test results in a browser.

HTMLReporter

Reporter that writes results as a standalone HTML file.

Produces a single HTML file with embedded CSS, requiring no external dependencies for viewing.

Attributes:

Name	Type	Description
output_dir		Directory to write report files to.

Source code in src/agentprobe/reporting/html.py

class HTMLReporter:
    """Reporter that writes results as a standalone HTML file.

    Produces a single HTML file with embedded CSS, requiring no external
    dependencies for viewing.

    Attributes:
        output_dir: Directory to write report files to.
    """

    def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
        """Initialize the HTML reporter.

        Args:
            output_dir: Directory for report output.
        """
        self._output_dir = Path(output_dir)

    @property
    def name(self) -> str:
        """Return the reporter name."""
        return "html"

    async def report(self, run: AgentRun) -> None:
        """Write the agent run as an HTML file.

        Args:
            run: The completed agent run.
        """
        self._output_dir.mkdir(parents=True, exist_ok=True)
        output_path = self._output_dir / f"report-{run.run_id}.html"

        content = self._build_html(run)
        output_path.write_text(content, encoding="utf-8")
        logger.info("HTML report written to %s", output_path)

    def _build_html(self, run: AgentRun) -> str:
        """Build the complete HTML content.

        Args:
            run: The completed agent run.

        Returns:
            The full HTML string.
        """
        rows = self._build_rows(run)
        overall_color = "#22c55e" if run.failed == 0 and run.errors == 0 else "#ef4444"

        return f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>AgentProbe Report — {html.escape(run.agent_name)}</title>
<style>
  body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
         max-width: 960px; margin: 2rem auto; padding: 0 1rem; background: #fafafa; color: #1a1a1a; }}
  h1 {{ margin-bottom: 0.5rem; }}
  .summary {{ display: flex; gap: 1.5rem; margin: 1rem 0; padding: 1rem;
              background: #fff; border-radius: 8px; border: 1px solid #e5e7eb; }}
  .summary .stat {{ text-align: center; }}
  .summary .stat .value {{ font-size: 1.5rem; font-weight: 700; }}
  .summary .stat .label {{ font-size: 0.8rem; color: #6b7280; text-transform: uppercase; }}
  table {{ width: 100%; border-collapse: collapse; background: #fff;
           border-radius: 8px; overflow: hidden; border: 1px solid #e5e7eb; }}
  th {{ background: #f9fafb; text-align: left; padding: 0.75rem 1rem;
       font-size: 0.8rem; text-transform: uppercase; color: #6b7280; border-bottom: 1px solid #e5e7eb; }}
  td {{ padding: 0.75rem 1rem; border-bottom: 1px solid #f3f4f6; }}
  .status {{ padding: 2px 8px; border-radius: 4px; color: #fff; font-size: 0.8rem; font-weight: 600; }}
</style>
</head>
<body>
<h1>AgentProbe Test Report</h1>
<p>Agent: <strong>{html.escape(run.agent_name)}</strong> —
   Status: <span style="color:{overall_color};font-weight:700">{html.escape(run.status.value)}</span></p>
<div class="summary">
  <div class="stat"><div class="value">{run.total_tests}</div><div class="label">Total</div></div>
  <div class="stat"><div class="value" style="color:#22c55e">{run.passed}</div><div class="label">Passed</div></div>
  <div class="stat"><div class="value" style="color:#ef4444">{run.failed}</div><div class="label">Failed</div></div>
  <div class="stat"><div class="value" style="color:#dc2626">{run.errors}</div><div class="label">Errors</div></div>
  <div class="stat"><div class="value">{run.duration_ms}ms</div><div class="label">Duration</div></div>
</div>
<table>
<thead><tr><th>Test</th><th>Status</th><th>Score</th><th>Duration</th><th>Details</th></tr></thead>
<tbody>
{rows}
</tbody>
</table>
</body>
</html>"""

    def _build_rows(self, run: AgentRun) -> str:
        """Build HTML table rows for test results.

        Args:
            run: The completed agent run.

        Returns:
            HTML string containing all table rows.
        """
        lines: list[str] = []
        for result in run.test_results:
            color = _STATUS_COLORS.get(result.status, "#6b7280")
            details = ""
            if result.error_message:
                details = html.escape(result.error_message[:100])
            elif result.eval_results:
                verdicts = [r.verdict.value for r in result.eval_results]
                details = html.escape(", ".join(verdicts))

            lines.append(
                f"<tr><td>{html.escape(result.test_name)}</td>"
                f'<td><span class="status" style="background:{color}">'
                f"{html.escape(result.status.value)}</span></td>"
                f"<td>{result.score:.2f}</td>"
                f"<td>{result.duration_ms}ms</td>"
                f"<td>{details}</td></tr>"
            )
        return "\n".join(lines)

name property

Return the reporter name.

__init__(output_dir='agentprobe-report')

Initialize the HTML reporter.

Parameters:

Name	Type	Description	Default
output_dir	str | Path	Directory for report output.	'agentprobe-report'

Source code in src/agentprobe/reporting/html.py

def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
    """Initialize the HTML reporter.

    Args:
        output_dir: Directory for report output.
    """
    self._output_dir = Path(output_dir)

report(run) async

Write the agent run as an HTML file.

Parameters:

Name	Type	Description	Default
run	AgentRun	The completed agent run.	required

Source code in src/agentprobe/reporting/html.py

async def report(self, run: AgentRun) -> None:
    """Write the agent run as an HTML file.

    Args:
        run: The completed agent run.
    """
    self._output_dir.mkdir(parents=True, exist_ok=True)
    output_path = self._output_dir / f"report-{run.run_id}.html"

    content = self._build_html(run)
    output_path.write_text(content, encoding="utf-8")
    logger.info("HTML report written to %s", output_path)

JSON Reporter

agentprobe.reporting.json_reporter

JSON file reporter for test results.

Writes the complete AgentRun as a JSON file for consumption by CI/CD pipelines and other tools.

JSONReporter

Reporter that writes results to a JSON file.

Attributes:

Name	Type	Description
output_dir		Directory to write report files to.

Source code in src/agentprobe/reporting/json_reporter.py

class JSONReporter:
    """Reporter that writes results to a JSON file.

    Attributes:
        output_dir: Directory to write report files to.
    """

    def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
        """Initialize the JSON reporter.

        Args:
            output_dir: Directory for report output.
        """
        self._output_dir = Path(output_dir)

    @property
    def name(self) -> str:
        """Return the reporter name."""
        return "json"

    async def report(self, run: AgentRun) -> None:
        """Write the agent run as a JSON file.

        Args:
            run: The completed agent run.
        """
        self._output_dir.mkdir(parents=True, exist_ok=True)
        output_path = self._output_dir / f"report-{run.run_id}.json"

        data = json.loads(run.model_dump_json())
        output_path.write_text(
            json.dumps(data, indent=2, ensure_ascii=False),
            encoding="utf-8",
        )
        logger.info("JSON report written to %s", output_path)

name property

Return the reporter name.

__init__(output_dir='agentprobe-report')

Initialize the JSON reporter.

Parameters:

Name	Type	Description	Default
output_dir	str | Path	Directory for report output.	'agentprobe-report'

Source code in src/agentprobe/reporting/json_reporter.py

def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
    """Initialize the JSON reporter.

    Args:
        output_dir: Directory for report output.
    """
    self._output_dir = Path(output_dir)

report(run) async

Write the agent run as a JSON file.

Parameters:

Name	Type	Description	Default
run	AgentRun	The completed agent run.	required

Source code in src/agentprobe/reporting/json_reporter.py

async def report(self, run: AgentRun) -> None:
    """Write the agent run as a JSON file.

    Args:
        run: The completed agent run.
    """
    self._output_dir.mkdir(parents=True, exist_ok=True)
    output_path = self._output_dir / f"report-{run.run_id}.json"

    data = json.loads(run.model_dump_json())
    output_path.write_text(
        json.dumps(data, indent=2, ensure_ascii=False),
        encoding="utf-8",
    )
    logger.info("JSON report written to %s", output_path)

JUnit XML Reporter

agentprobe.reporting.junit

JUnit XML reporter for test results.

Generates JUnit-compatible XML output suitable for CI/CD systems like Jenkins, GitHub Actions, and GitLab CI.

JUnitReporter

Reporter that writes results as JUnit XML.

Produces a standard JUnit XML file that can be consumed by CI/CD pipelines for test reporting and status visualization.

Attributes:

Name	Type	Description
output_dir		Directory to write report files to.

Source code in src/agentprobe/reporting/junit.py

class JUnitReporter:
    """Reporter that writes results as JUnit XML.

    Produces a standard JUnit XML file that can be consumed by CI/CD
    pipelines for test reporting and status visualization.

    Attributes:
        output_dir: Directory to write report files to.
    """

    def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
        """Initialize the JUnit reporter.

        Args:
            output_dir: Directory for report output.
        """
        self._output_dir = Path(output_dir)

    @property
    def name(self) -> str:
        """Return the reporter name."""
        return "junit"

    async def report(self, run: AgentRun) -> None:
        """Write the agent run as a JUnit XML file.

        Args:
            run: The completed agent run.
        """
        self._output_dir.mkdir(parents=True, exist_ok=True)
        output_path = self._output_dir / f"report-{run.run_id}.xml"

        root = self._build_xml(run)
        tree = ET.ElementTree(root)
        ET.indent(tree, space="  ")
        tree.write(str(output_path), encoding="unicode", xml_declaration=True)
        logger.info("JUnit XML report written to %s", output_path)

    def _build_xml(self, run: AgentRun) -> ET.Element:
        """Build the JUnit XML element tree.

        Args:
            run: The completed agent run.

        Returns:
            The root XML element.
        """
        testsuite = ET.Element("testsuite")
        testsuite.set("name", run.agent_name)
        testsuite.set("tests", str(run.total_tests))
        testsuite.set("failures", str(run.failed))
        testsuite.set("errors", str(run.errors))
        testsuite.set("skipped", str(run.skipped))
        testsuite.set("time", f"{run.duration_ms / 1000:.3f}")

        for result in run.test_results:
            testcase = self._build_testcase(result, run.agent_name)
            testsuite.append(testcase)

        return testsuite

    def _build_testcase(self, result: TestResult, suite_name: str) -> ET.Element:
        """Build a testcase XML element.

        Args:
            result: A single test result.
            suite_name: The parent test suite name.

        Returns:
            A testcase XML element.
        """
        testcase = ET.Element("testcase")
        testcase.set("name", result.test_name)
        testcase.set("classname", suite_name)
        testcase.set("time", f"{result.duration_ms / 1000:.3f}")

        if result.status == TestStatus.FAILED:
            failure = ET.SubElement(testcase, "failure")
            failure.set("message", result.error_message or "Test failed")
            failure.set("type", "AssertionError")
            if result.eval_results:
                failure.text = "\n".join(
                    f"{er.evaluator_name}: {er.verdict.value} ({er.score:.2f}) - {er.reason}"
                    for er in result.eval_results
                )

        elif result.status == TestStatus.ERROR:
            error = ET.SubElement(testcase, "error")
            error.set("message", result.error_message or "Test error")
            error.set("type", "RuntimeError")

        elif result.status == TestStatus.SKIPPED:
            skipped = ET.SubElement(testcase, "skipped")
            skipped.set("message", result.error_message or "Test skipped")

        return testcase

name property

Return the reporter name.

__init__(output_dir='agentprobe-report')

Initialize the JUnit reporter.

Parameters:

Name	Type	Description	Default
output_dir	str | Path	Directory for report output.	'agentprobe-report'

Source code in src/agentprobe/reporting/junit.py

def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
    """Initialize the JUnit reporter.

    Args:
        output_dir: Directory for report output.
    """
    self._output_dir = Path(output_dir)

report(run) async

Write the agent run as a JUnit XML file.

Parameters:

Name	Type	Description	Default
run	AgentRun	The completed agent run.	required

Source code in src/agentprobe/reporting/junit.py

async def report(self, run: AgentRun) -> None:
    """Write the agent run as a JUnit XML file.

    Args:
        run: The completed agent run.
    """
    self._output_dir.mkdir(parents=True, exist_ok=True)
    output_path = self._output_dir / f"report-{run.run_id}.xml"

    root = self._build_xml(run)
    tree = ET.ElementTree(root)
    ET.indent(tree, space="  ")
    tree.write(str(output_path), encoding="unicode", xml_declaration=True)
    logger.info("JUnit XML report written to %s", output_path)

Markdown Reporter

agentprobe.reporting.markdown

Markdown reporter for test results.

Generates a Markdown file with tables and summary sections suitable for rendering in GitHub, GitLab, or documentation systems.

MarkdownReporter

Reporter that writes results as a Markdown file.

Produces a Markdown document with a summary section and a results table, compatible with GitHub, GitLab, and other renderers.

Attributes:

Name	Type	Description
output_dir		Directory to write report files to.

Source code in src/agentprobe/reporting/markdown.py

class MarkdownReporter:
    """Reporter that writes results as a Markdown file.

    Produces a Markdown document with a summary section and a results
    table, compatible with GitHub, GitLab, and other renderers.

    Attributes:
        output_dir: Directory to write report files to.
    """

    def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
        """Initialize the Markdown reporter.

        Args:
            output_dir: Directory for report output.
        """
        self._output_dir = Path(output_dir)

    @property
    def name(self) -> str:
        """Return the reporter name."""
        return "markdown"

    async def report(self, run: AgentRun) -> None:
        """Write the agent run as a Markdown file.

        Args:
            run: The completed agent run.
        """
        self._output_dir.mkdir(parents=True, exist_ok=True)
        output_path = self._output_dir / f"report-{run.run_id}.md"

        content = self._build_markdown(run)
        output_path.write_text(content, encoding="utf-8")
        logger.info("Markdown report written to %s", output_path)

    def _build_markdown(self, run: AgentRun) -> str:
        """Build the complete Markdown content.

        Args:
            run: The completed agent run.

        Returns:
            The full Markdown string.
        """
        lines: list[str] = []

        lines.append(f"# AgentProbe Test Report — {run.agent_name}")
        lines.append("")
        lines.append(f"**Status:** {run.status.value}")
        lines.append("")

        # Summary
        lines.append("## Summary")
        lines.append("")
        lines.append("| Metric | Value |")
        lines.append("|--------|-------|")
        lines.append(f"| Total Tests | {run.total_tests} |")
        lines.append(f"| Passed | {run.passed} |")
        lines.append(f"| Failed | {run.failed} |")
        lines.append(f"| Errors | {run.errors} |")
        lines.append(f"| Skipped | {run.skipped} |")
        lines.append(f"| Duration | {run.duration_ms}ms |")
        lines.append("")

        # Results table
        if run.test_results:
            lines.append("## Results")
            lines.append("")
            lines.append("| Test | Status | Score | Duration | Details |")
            lines.append("|------|--------|-------|----------|---------|")

            for result in run.test_results:
                status_label = _STATUS_EMOJI.get(result.status.value, result.status.value)
                details = ""
                if result.error_message:
                    details = result.error_message[:80]
                elif result.eval_results:
                    verdicts = [r.verdict.value for r in result.eval_results]
                    details = ", ".join(verdicts)

                lines.append(
                    f"| {result.test_name} | {status_label} | {result.score:.2f} "
                    f"| {result.duration_ms}ms | {details} |"
                )
            lines.append("")

        return "\n".join(lines)

name property

Return the reporter name.

__init__(output_dir='agentprobe-report')

Initialize the Markdown reporter.

Parameters:

Name	Type	Description	Default
output_dir	str | Path	Directory for report output.	'agentprobe-report'

Source code in src/agentprobe/reporting/markdown.py

def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
    """Initialize the Markdown reporter.

    Args:
        output_dir: Directory for report output.
    """
    self._output_dir = Path(output_dir)

report(run) async

Write the agent run as a Markdown file.

Parameters:

Name	Type	Description	Default
run	AgentRun	The completed agent run.	required

Source code in src/agentprobe/reporting/markdown.py

async def report(self, run: AgentRun) -> None:
    """Write the agent run as a Markdown file.

    Args:
        run: The completed agent run.
    """
    self._output_dir.mkdir(parents=True, exist_ok=True)
    output_path = self._output_dir / f"report-{run.run_id}.md"

    content = self._build_markdown(run)
    output_path.write_text(content, encoding="utf-8")
    logger.info("Markdown report written to %s", output_path)

CSV Reporter

agentprobe.reporting.csv_reporter

CSV reporter for test results.

Writes test results as a CSV file for easy import into spreadsheets and data analysis tools.

CSVReporter

Reporter that writes results as a CSV file.

Produces a CSV file with one row per test result, suitable for import into spreadsheets and data analysis pipelines.

Attributes:

Name	Type	Description
output_dir		Directory to write report files to.

Source code in src/agentprobe/reporting/csv_reporter.py

class CSVReporter:
    """Reporter that writes results as a CSV file.

    Produces a CSV file with one row per test result, suitable for
    import into spreadsheets and data analysis pipelines.

    Attributes:
        output_dir: Directory to write report files to.
    """

    def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
        """Initialize the CSV reporter.

        Args:
            output_dir: Directory for report output.
        """
        self._output_dir = Path(output_dir)

    @property
    def name(self) -> str:
        """Return the reporter name."""
        return "csv"

    async def report(self, run: AgentRun) -> None:
        """Write the agent run as a CSV file.

        Args:
            run: The completed agent run.
        """
        self._output_dir.mkdir(parents=True, exist_ok=True)
        output_path = self._output_dir / f"report-{run.run_id}.csv"

        content = self._build_csv(run)
        output_path.write_text(content, encoding="utf-8")
        logger.info("CSV report written to %s", output_path)

    def _build_csv(self, run: AgentRun) -> str:
        """Build the CSV content as a string.

        Args:
            run: The completed agent run.

        Returns:
            The complete CSV string with headers and data rows.
        """
        output = io.StringIO(newline="")
        writer = csv.writer(output)
        writer.writerow(_CSV_HEADERS)

        for result in run.test_results:
            eval_verdicts = ""
            if result.eval_results:
                verdicts = [r.verdict.value for r in result.eval_results]
                eval_verdicts = "; ".join(verdicts)

            writer.writerow(
                [
                    result.test_name,
                    result.status.value,
                    f"{result.score:.4f}",
                    str(result.duration_ms),
                    result.error_message or "",
                    eval_verdicts,
                ]
            )

        return output.getvalue()

name property

Return the reporter name.

__init__(output_dir='agentprobe-report')

Initialize the CSV reporter.

Parameters:

Name	Type	Description	Default
output_dir	str | Path	Directory for report output.	'agentprobe-report'

Source code in src/agentprobe/reporting/csv_reporter.py

def __init__(self, output_dir: str | Path = "agentprobe-report") -> None:
    """Initialize the CSV reporter.

    Args:
        output_dir: Directory for report output.
    """
    self._output_dir = Path(output_dir)

report(run) async

Write the agent run as a CSV file.

Parameters:

Name	Type	Description	Default
run	AgentRun	The completed agent run.	required

Source code in src/agentprobe/reporting/csv_reporter.py

async def report(self, run: AgentRun) -> None:
    """Write the agent run as a CSV file.

    Args:
        run: The completed agent run.
    """
    self._output_dir.mkdir(parents=True, exist_ok=True)
    output_path = self._output_dir / f"report-{run.run_id}.csv"

    content = self._build_csv(run)
    output_path.write_text(content, encoding="utf-8")
    logger.info("CSV report written to %s", output_path)