Chapter 8: TestNG Listeners

Built-in Listeners

TestNG provides a powerful listener framework that allows you to hook into the test execution lifecycle and customize the behavior of your tests. Listeners can be used for various purposes, such as logging, reporting, and modifying test behavior.

ITestListener

The ITestListener interface is one of the most commonly used listeners in TestNG. It provides methods that are called during the test execution lifecycle:

import org.testng.ITestContext;
import org.testng.ITestListener;
import org.testng.ITestResult;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

@Listeners(TestExecutionListener.class)
public class TestListenerExample {

    @Test
    public void testSuccess() {
        System.out.println("This test will pass");
    }

    @Test
    public void testFailure() {
        System.out.println("This test will fail");
        throw new RuntimeException("Deliberate failure");
    }

    @Test(enabled = false)
    public void testSkipped() {
        System.out.println("This test will be skipped");
    }
}

class TestExecutionListener implements ITestListener {

    @Override
    public void onTestStart(ITestResult result) {
        System.out.println("Test started: " + result.getName());
    }

    @Override
    public void onTestSuccess(ITestResult result) {
        System.out.println("Test succeeded: " + result.getName());
    }

    @Override
    public void onTestFailure(ITestResult result) {
        System.out.println("Test failed: " + result.getName());
        System.out.println("Exception: " + result.getThrowable().getMessage());
    }

    @Override
    public void onTestSkipped(ITestResult result) {
        System.out.println("Test skipped: " + result.getName());
    }

    @Override
    public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
        System.out.println("Test failed but within success percentage: " + result.getName());
    }

    @Override
    public void onStart(ITestContext context) {
        System.out.println("Test execution started: " + context.getName());
    }

    @Override
    public void onFinish(ITestContext context) {
        System.out.println("Test execution finished: " + context.getName());
        System.out.println("Passed tests: " + context.getPassedTests().size());
        System.out.println("Failed tests: " + context.getFailedTests().size());
        System.out.println("Skipped tests: " + context.getSkippedTests().size());
    }
}

ISuiteListener

The ISuiteListener interface allows you to hook into the suite execution lifecycle:

import org.testng.ISuite;
import org.testng.ISuiteListener;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

@Listeners(SuiteExecutionListener.class)
public class SuiteListenerExample {

    @Test
    public void test1() {
        System.out.println("Running test1");
    }

    @Test
    public void test2() {
        System.out.println("Running test2");
    }
}

class SuiteExecutionListener implements ISuiteListener {

    @Override
    public void onStart(ISuite suite) {
        System.out.println("Suite started: " + suite.getName());
    }

    @Override
    public void onFinish(ISuite suite) {
        System.out.println("Suite finished: " + suite.getName());
        System.out.println("Total tests run: " + suite.getAllMethods().size());
    }
}

IInvokedMethodListener

The IInvokedMethodListener interface allows you to hook into the method invocation lifecycle:

import org.testng.IInvokedMethod;
import org.testng.IInvokedMethodListener;
import org.testng.ITestResult;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

@Listeners(MethodInvocationListener.class)
public class InvokedMethodListenerExample {

    @Test
    public void testMethod() {
        System.out.println("Running test method");
    }
}

class MethodInvocationListener implements IInvokedMethodListener {

    @Override
    public void beforeInvocation(IInvokedMethod method, ITestResult testResult) {
        System.out.println("Before invoking: " + method.getTestMethod().getMethodName());
    }

    @Override
    public void afterInvocation(IInvokedMethod method, ITestResult testResult) {
        System.out.println("After invoking: " + method.getTestMethod().getMethodName());
        System.out.println("Execution time: " +
                          (testResult.getEndMillis() - testResult.getStartMillis()) + " ms");
    }
}

Custom Listeners

You can create custom listeners by implementing one or more of the TestNG listener interfaces. This allows you to add specific behavior to your test execution.

Creating a Custom Listener

import org.testng.ITestContext;
import org.testng.ITestListener;
import org.testng.ITestResult;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

import java.util.ArrayList;
import java.util.List;

@Listeners(TestMetricsListener.class)
public class CustomListenerExample {

    @Test
    public void fastTest() throws InterruptedException {
        Thread.sleep(100); // Simulate work
    }

    @Test
    public void mediumTest() throws InterruptedException {
        Thread.sleep(500); // Simulate work
    }

    @Test
    public void slowTest() throws InterruptedException {
        Thread.sleep(1000); // Simulate work
    }
}

class TestMetricsListener implements ITestListener {

    private List<TestMetric> metrics = new ArrayList<>();

    @Override
    public void onTestStart(ITestResult result) {
        result.setAttribute("startTime", System.currentTimeMillis());
    }

    @Override
    public void onTestSuccess(ITestResult result) {
        long startTime = (Long) result.getAttribute("startTime");
        long endTime = System.currentTimeMillis();
        long duration = endTime - startTime;

        metrics.add(new TestMetric(
            result.getName(),
            duration,
            "PASS"
        ));
    }

    @Override
    public void onTestFailure(ITestResult result) {
        long startTime = (Long) result.getAttribute("startTime");
        long endTime = System.currentTimeMillis();
        long duration = endTime - startTime;

        metrics.add(new TestMetric(
            result.getName(),
            duration,
            "FAIL"
        ));
    }

    @Override
    public void onFinish(ITestContext context) {
        System.out.println("Test Execution Metrics:");
        System.out.println("----------------------");

        // Sort metrics by duration (descending)
        metrics.sort((m1, m2) -> Long.compare(m2.duration(), m1.duration()));

        for (TestMetric metric : metrics) {
            System.out.printf("%-20s %-10s %5d ms%n",
                             metric.testName(), metric.result(), metric.duration());
        }

        // Calculate statistics
        long totalDuration = metrics.stream()
                .mapToLong(TestMetric::duration)
                .sum();

        double averageDuration = metrics.stream()
                .mapToLong(TestMetric::duration)
                .average()
                .orElse(0);

        System.out.println("----------------------");
        System.out.println("Total tests: " + metrics.size());
        System.out.println("Total duration: " + totalDuration + " ms");
        System.out.println("Average duration: " + String.format("%.2f", averageDuration) + " ms");
    }

    // Other methods of ITestListener are left empty
    @Override public void onTestSkipped(ITestResult result) {}
    @Override public void onTestFailedButWithinSuccessPercentage(ITestResult result) {}
    @Override public void onStart(ITestContext context) {}
}

// Using Java 21 record for test metrics
record TestMetric(String testName, long duration, String result) {}

Combining Multiple Listeners

You can implement multiple listener interfaces in a single class:

import org.testng.*;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

@Listeners(CombinedListener.class)
public class MultipleListenerExample {

    @Test
    public void testMethod() {
        System.out.println("Running test method");
    }
}

class CombinedListener implements ITestListener, ISuiteListener, IInvokedMethodListener {

    // ITestListener methods
    @Override
    public void onTestStart(ITestResult result) {
        System.out.println("Test started: " + result.getName());
    }

    @Override
    public void onTestSuccess(ITestResult result) {
        System.out.println("Test succeeded: " + result.getName());
    }

    @Override
    public void onTestFailure(ITestResult result) {
        System.out.println("Test failed: " + result.getName());
    }

    @Override
    public void onTestSkipped(ITestResult result) {
        System.out.println("Test skipped: " + result.getName());
    }

    @Override
    public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
        System.out.println("Test failed but within success percentage: " + result.getName());
    }

    @Override
    public void onStart(ITestContext context) {
        System.out.println("Test execution started: " + context.getName());
    }

    @Override
    public void onFinish(ITestContext context) {
        System.out.println("Test execution finished: " + context.getName());
    }

    // ISuiteListener methods
    @Override
    public void onStart(ISuite suite) {
        System.out.println("Suite started: " + suite.getName());
    }

    @Override
    public void onFinish(ISuite suite) {
        System.out.println("Suite finished: " + suite.getName());
    }

    // IInvokedMethodListener methods
    @Override
    public void beforeInvocation(IInvokedMethod method, ITestResult testResult) {
        System.out.println("Before invoking: " + method.getTestMethod().getMethodName());
    }

    @Override
    public void afterInvocation(IInvokedMethod method, ITestResult testResult) {
        System.out.println("After invoking: " + method.getTestMethod().getMethodName());
    }
}

Reporting Listeners

TestNG provides listeners specifically for customizing test reports.

IReporter

The IReporter interface allows you to generate custom reports after all tests have been run:

import org.testng.*;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
import org.testng.xml.XmlSuite;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;

@Listeners(CustomReporter.class)
public class ReporterExample {

    @Test
    public void testSuccess() {
        System.out.println("This test will pass");
    }

    @Test
    public void testFailure() {
        System.out.println("This test will fail");
        throw new RuntimeException("Deliberate failure");
    }
}

class CustomReporter implements IReporter {

    @Override
    public void generateReport(List<XmlSuite> xmlSuites, List<ISuite> suites, String outputDirectory) {
        // Create a simple HTML report
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputDirectory + "/custom-report.html"))) {
            writer.write("<html><head><title>Custom TestNG Report</title></head><body>");
            writer.write("<h1>Custom TestNG Report</h1>");

            for (ISuite suite : suites) {
                writer.write("<h2>Suite: " + suite.getName() + "</h2>");

                for (ISuiteResult result : suite.getResults().values()) {
                    ITestContext context = result.getTestContext();

                    writer.write("<h3>Test: " + context.getName() + "</h3>");

                    // Write passed tests
                    writer.write("<h4>Passed Tests</h4>");
                    writer.write("<ul>");
                    for (ITestResult testResult : context.getPassedTests().getAllResults()) {
                        writer.write("<li>" + testResult.getName() + " - " +
                                    (testResult.getEndMillis() - testResult.getStartMillis()) + " ms</li>");
                    }
                    writer.write("</ul>");

                    // Write failed tests
                    writer.write("<h4>Failed Tests</h4>");
                    writer.write("<ul>");
                    for (ITestResult testResult : context.getFailedTests().getAllResults()) {
                        writer.write("<li>" + testResult.getName() + " - " +
                                    testResult.getThrowable().getMessage() + "</li>");
                    }
                    writer.write("</ul>");

                    // Write skipped tests
                    writer.write("<h4>Skipped Tests</h4>");
                    writer.write("<ul>");
                    for (ITestResult testResult : context.getSkippedTests().getAllResults()) {
                        writer.write("<li>" + testResult.getName() + "</li>");
                    }
                    writer.write("</ul>");
                }
            }

            writer.write("</body></html>");

            System.out.println("Custom report generated at: " + outputDirectory + "/custom-report.html");
        } catch (IOException e) {
            System.err.println("Error generating report: " + e.getMessage());
        }
    }
}

Extending EmailableReporter

You can extend TestNG's built-in reporters to customize them:

import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
import org.testng.reporters.EmailableReporter;

import java.util.List;

@Listeners(CustomEmailableReporter.class)
public class EmailableReporterExample {

    @Test
    public void testSuccess() {
        System.out.println("This test will pass");
    }

    @Test
    public void testFailure() {
        System.out.println("This test will fail");
        throw new RuntimeException("Deliberate failure");
    }
}

class CustomEmailableReporter extends EmailableReporter {

    @Override
    protected String getReportTitle() {
        return "Custom Emailable Report";
    }

    @Override
    protected String getReportName() {
        return "Custom Emailable Report";
    }

    // Override other methods as needed to customize the report
}

Annotation Transformers

TestNG provides annotation transformers that allow you to modify test annotations at runtime.

IAnnotationTransformer

The IAnnotationTransformer interface allows you to modify @Test annotations:

import org.testng.IAnnotationTransformer;
import org.testng.annotations.ITestAnnotation;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

import java.lang.reflect.Constructor;
import java.lang.reflect.Method;

@Listeners(TimeoutTransformer.class)
public class AnnotationTransformerExample {

    @Test
    public void testWithoutTimeout() {
        System.out.println("This test will have a timeout added dynamically");
        // Test logic
    }

    @Test(timeOut = 5000)
    public void testWithTimeout() {
        System.out.println("This test already has a timeout");
        // Test logic
    }
}

class TimeoutTransformer implements IAnnotationTransformer {

    @Override
    public void transform(ITestAnnotation annotation,
                         Class testClass,
                         Constructor testConstructor,
                         Method testMethod) {
        // Add a timeout to all test methods that don't already have one
        if (annotation.getTimeOut() == 0) {
            annotation.setTimeOut(2000); // 2 seconds timeout
        }
    }
}