Back to Chapters

Chapter 11: Reporting and Logging

Built-in Reporting in TestNG

TestNG provides several built-in reporting mechanisms to help you understand test results and diagnose issues.

Default HTML Report

By default, TestNG generates an HTML report after test execution. This report is typically located in the test-output directory and provides a summary of test results, including passed, failed, and skipped tests.

import org.testng.annotations.Test;

public class SimpleReportExample {

    @Test
    public void testSuccess() {
        // This test will pass
        assert true;
    }

    @Test
    public void testFailure() {
        // This test will fail
        assert false : "Deliberate failure";
    }

    @Test(enabled = false)
    public void testSkipped() {
        // This test will be skipped
    }
}

When you run this test class, TestNG will generate an HTML report that includes:

  1. A summary of test results
  2. Details of each test method
  3. Execution time for each test
  4. Stack traces for failed tests

Emailable Report

TestNG also generates an emailable report that can be easily shared with team members:

import org.testng.annotations.Test;

public class EmailableReportExample {

    @Test(groups = {"smoke"})
    public void testFeature1() {
        // Test logic
    }

    @Test(groups = {"regression"})
    public void testFeature2() {
        // Test logic
    }

    @Test(groups = {"smoke", "regression"})
    public void testFeature3() {
        // Test logic
    }
}

The emailable report provides a concise summary of test results that can be easily shared via email.

JUnit XML Report

TestNG can generate JUnit XML reports that are compatible with continuous integration tools like Jenkins:

<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
<suite name="JUnitXMLSuite">
    <listeners>
        <listener class-name="org.testng.reporters.JUnitXMLReporter"/>
    </listeners>
    <test name="JUnitXMLTest">
        <classes>
            <class name="com.example.JUnitXMLReportExample"/>
        </classes>
    </test>
</suite>
import org.testng.annotations.Test;

public class JUnitXMLReportExample {

    @Test
    public void testMethod1() {
        // Test logic
    }

    @Test
    public void testMethod2() {
        // Test logic
    }
}

Custom Reporting

You can create custom reports to meet your specific needs.

Implementing IReporter

The IReporter interface allows you to generate custom reports:

import org.testng.IReporter;
import org.testng.ISuite;
import org.testng.ISuiteResult;
import org.testng.ITestContext;
import org.testng.ITestResult;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
import org.testng.xml.XmlSuite;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
import java.util.Map;

@Listeners(CustomReporter.class)
public class CustomReportExample {

    @Test
    public void testSuccess() {
        // This test will pass
    }

    @Test
    public void testFailure() {
        // This test will fail
        assert false : "Deliberate failure";
    }
}

class CustomReporter implements IReporter {

    @Override
    public void generateReport(List<XmlSuite> xmlSuites, List<ISuite> suites, String outputDirectory) {
        // Create a custom report
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputDirectory + "/custom-report.html"))) {
            writer.write("<html><head><title>Custom TestNG Report</title></head><body>");
            writer.write("<h1>Custom TestNG Report</h1>");

            for (ISuite suite : suites) {
                writer.write("<h2>Suite: " + suite.getName() + "</h2>");

                Map<String, ISuiteResult> results = suite.getResults();

                for (ISuiteResult result : results.values()) {
                    ITestContext context = result.getTestContext();

                    writer.write("<h3>Test: " + context.getName() + "</h3>");

                    // Write passed tests
                    writer.write("<h4>Passed Tests</h4>");
                    writer.write("<ul>");
                    for (ITestResult testResult : context.getPassedTests().getAllResults()) {
                        writer.write("<li>" + testResult.getName() + " - " +
                                    (testResult.getEndMillis() - testResult.getStartMillis()) + " ms</li>");
                    }
                    writer.write("</ul>");

                    // Write failed tests
                    writer.write("<h4>Failed Tests</h4>");
                    writer.write("<ul>");
                    for (ITestResult testResult : context.getFailedTests().getAllResults()) {
                        writer.write("<li>" + testResult.getName() + " - " +
                                    testResult.getThrowable().getMessage() + "</li>");
                    }
                    writer.write("</ul>");

                    // Write skipped tests
                    writer.write("<h4>Skipped Tests</h4>");
                    writer.write("<ul>");
                    for (ITestResult testResult : context.getSkippedTests().getAllResults()) {
                        writer.write("<li>" + testResult.getName() + "</li>");
                    }
                    writer.write("</ul>");
                }
            }

            writer.write("</body></html>");

            System.out.println("Custom report generated at: " + outputDirectory + "/custom-report.html");
        } catch (IOException e) {
            System.err.println("Error generating report: " + e.getMessage());
        }
    }
}

Extending EmailableReporter

You can extend TestNG's built-in reporters to customize them:

import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
import org.testng.reporters.EmailableReporter;

import java.util.List;

@Listeners(CustomEmailableReporter.class)
public class CustomEmailableReportExample {

    @Test
    public void testMethod1() {
        // Test logic
    }

    @Test
    public void testMethod2() {
        // Test logic
        assert false : "Deliberate failure";
    }
}

class CustomEmailableReporter extends EmailableReporter {

    @Override
    protected String getReportTitle() {
        return "Custom Emailable Report";
    }

    @Override
    protected String getReportName() {
        return "Custom Emailable Report";
    }

    // Override other methods as needed to customize the report
}

Logging in TestNG

Proper logging is essential for diagnosing issues in your tests.

Using SLF4J with TestNG

SLF4J (Simple Logging Facade for Java) is a popular logging facade that works well with TestNG:

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;

public class SLF4JLoggingExample {

    private static final Logger logger = LoggerFactory.getLogger(SLF4JLoggingExample.class);

    @Test
    public void testWithLogging() {
        logger.info("Starting test");

        // Test setup
        logger.debug("Setting up test data");

        // Test execution
        logger.debug("Executing test");

        // Test verification
        logger.debug("Verifying results");

        logger.info("Test completed");
    }

    @Test
    public void testWithErrorLogging() {
        logger.info("Starting test with error");

        try {
            // Code that might throw an exception
            int result = 10 / 0;
        } catch (Exception e) {
            logger.error("Error during test execution", e);
            throw e;
        }
    }
}

Using Log4j2 with TestNG

Log4j2 is another popular logging framework that can be used with TestNG:

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.testng.annotations.Test;

public class Log4j2LoggingExample {

    private static final Logger logger = LogManager.getLogger(Log4j2LoggingExample.class);

    @Test
    public void testWithLogging() {
        logger.info("Starting test");

        // Test setup
        logger.debug("Setting up test data");

        // Test execution
        logger.debug("Executing test");

        // Test verification
        logger.debug("Verifying results");

        logger.info("Test completed");
    }

    @Test
    public void testWithErrorLogging() {
        logger.info("Starting test with error");

        try {
            // Code that might throw an exception
            int result = 10 / 0;
        } catch (Exception e) {
            logger.error("Error during test execution", e);
            throw e;
        }
    }
}

Creating a Logging Listener

You can create a TestNG listener that logs test events:

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.ITestContext;
import org.testng.ITestListener;
import org.testng.ITestResult;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

@Listeners(LoggingListener.class)
public class LoggingListenerExample {

    @Test
    public void testMethod1() {
        // Test logic
    }

    @Test
    public void testMethod2() {
        // Test logic that fails
        assert false : "Deliberate failure";
    }
}

class LoggingListener implements ITestListener {

    private static final Logger logger = LoggerFactory.getLogger(LoggingListener.class);

    @Override
    public void onTestStart(ITestResult result) {
        logger.info("Starting test: {}", result.getName());
    }

    @Override
    public void onTestSuccess(ITestResult result) {
        logger.info("Test passed: {}", result.getName());
    }

    @Override
    public void onTestFailure(ITestResult result) {
        logger.error("Test failed: {}", result.getName());
        logger.error("Exception: ", result.getThrowable());
    }

    @Override
    public void onTestSkipped(ITestResult result) {
        logger.warn("Test skipped: {}", result.getName());
    }

    @Override
    public void onStart(ITestContext context) {
        logger.info("Starting test execution: {}", context.getName());
    }

    @Override
    public void onFinish(ITestContext context) {
        logger.info("Finished test execution: {}", context.getName());
        logger.info("Passed tests: {}", context.getPassedTests().size());
        logger.info("Failed tests: {}", context.getFailedTests().size());
        logger.info("Skipped tests: {}", context.getSkippedTests().size());
    }

    @Override
    public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
        logger.warn("Test failed but within success percentage: {}", result.getName());
    }
}

Advanced Reporting and Logging

Let's explore some advanced reporting and logging techniques.

Generating Reports with Screenshots

For UI tests, it's often useful to include screenshots in your reports:

import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriver;
import org.testng.ITestResult;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;

public class ScreenshotReportExample {

    private WebDriver driver;

    @BeforeMethod
    public void setup() {
        // Initialize WebDriver
        // driver = new ChromeDriver();
    }

    @Test
    public void testWithScreenshot() {
        // Test logic
    }

    @AfterMethod
    public void tearDown(ITestResult result) {
        if (result.getStatus() == ITestResult.FAILURE) {
            // Take screenshot
            File screenshot = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);

            // Save screenshot
            String screenshotName = result.getName() + "_" + System.currentTimeMillis() + ".png";
            Path destination = Paths.get("test-output/screenshots/" + screenshotName);

            try {
                Files.createDirectories(destination.getParent());
                Files.copy(screenshot.toPath(), destination, StandardCopyOption.REPLACE_EXISTING);
                System.out.println("Screenshot saved: " + destination);
            } catch (IOException e) {
                System.err.println("Error saving screenshot: " + e.getMessage());
            }
        }

        // Quit WebDriver
        if (driver != null) {
            driver.quit();
        }
    }
}

Generating Reports with Test Data

Including test data in your reports can help with debugging:

import org.testng.ITestContext;
import org.testng.ITestListener;
import org.testng.ITestResult;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

import java.util.Arrays;

@Listeners(TestDataListener.class)
public class TestDataReportExample {

    @DataProvider(name = "testData")
    public Object[][] createData() {
        return new Object[][] {
            {"data1", 10},
            {"data2", 20},
            {"data3", 30}
        };
    }

    @Test(dataProvider = "testData")
    public void testWithData(String str, int num) {
        // Test logic using the data
        System.out.println("Testing with: " + str + ", " + num);

        // Simulate a failure for one data set
        if (num == 20) {
            assert false : "Deliberate failure for data: " + str + ", " + num;
        }
    }
}

class TestDataListener implements ITestListener {

    @Override
    public void onTestStart(ITestResult result) {
        System.out.println("Starting test: " + result.getName() +
                           " with parameters: " + Arrays.toString(result.getParameters()));
    }

    @Override
    public void onTestFailure(ITestResult result) {
        System.err.println("Test failed: " + result.getName() +
                           " with parameters: " + Arrays.toString(result.getParameters()));
        System.err.println("Exception: " + result.getThrowable().getMessage());
    }

    // Other methods of ITestListener are left empty for brevity
    @Override public void onTestSuccess(ITestResult result) {}
    @Override public void onTestSkipped(ITestResult result) {}
    @Override public void onTestFailedButWithinSuccessPercentage(ITestResult result) {}
    @Override public void onStart(ITestContext context) {}
    @Override public void onFinish(ITestContext context) {}
}