Do not report flaky failures in upstream tests.

base/test/launcher/teamcity_reporter.cc

@@ -13,6 +13,7 @@
 #include "base/environment.h"
 #include "base/files/file_path.h"
 #include "base/path_service.h"
+#include "base/strings/string_util.h"
 #include "base/test/launcher/test_result.h"
 
 namespace base {
@@ -28,6 +29,11 @@ constexpr char kTestLauncherEnableTeamcityReporter[] =
 constexpr char kTestLauncherDisableTeamcityReporter[] =
     "test-launcher-disable-teamcity-reporter";
 
+// This switch enables the TeamcityReporter to ignore preliminary test failures
+// when test retries are enabled, reporting only the final result of each test.
+constexpr char kTestLauncherTeamcityReporterIgnorePreliminaryFailures[] =
+    "test-launcher-teamcity-reporter-ignore-preliminary-failures";
+
 // Returns the name of the current executable, excluding the extension.
 std::string GetExecutableName() {
   return PathService::CheckedGet(FILE_EXE)
@@ -38,6 +44,18 @@ std::string GetExecutableName() {
 
 }  // namespace
 
+constexpr std::string_view TeamcityReporter::kPreliminaryFailureIgnoreMessage =
+    "Failure ignored, expecting a retry";
+
+constexpr std::string_view TeamcityReporter::kTestSkippedIgnoreMessage =
+    "Skipped, possibly because of a previous failure";
+
+constexpr std::string_view TeamcityReporter::kNotRetriedMessage =
+    "NOT_RETRIED (suite early exit)";
+
+TeamcityReporter::TestFailure::TestFailure() = default;
+TeamcityReporter::TestFailure::~TestFailure() = default;
+
 // static
 std::unique_ptr<TeamcityReporter> TeamcityReporter::MaybeCreate() {
   const auto environment = Environment::Create();
@@ -51,15 +69,21 @@ std::unique_ptr<TeamcityReporter> TeamcityReporter::MaybeCreate() {
       command_line->HasSwitch(kTestLauncherDisableTeamcityReporter);
 
   if (should_enable && !should_disable) {
-    return std::make_unique<TeamcityReporter>(std::cout, GetExecutableName());
+    const bool ignore_preliminary_failures = command_line->HasSwitch(
+        kTestLauncherTeamcityReporterIgnorePreliminaryFailures);
+    return std::make_unique<TeamcityReporter>(std::cout, GetExecutableName(),
+                                              ignore_preliminary_failures);
   }
 
   return nullptr;
 }
 
 TeamcityReporter::TeamcityReporter(std::ostream& ostream,
-                                   std::string suite_name)
-    : tsm_(ostream), suite_name_(std::move(suite_name)) {
+                                   std::string suite_name,
+                                   bool ignore_preliminary_failures)
+    : tsm_(ostream),
+      suite_name_(std::move(suite_name)),
+      ignore_preliminary_failures_(ignore_preliminary_failures) {
   LogSuiteStarted();
 }
 
@@ -67,9 +91,10 @@ TeamcityReporter::~TeamcityReporter() {
   LogSuiteFinished();
 }
 
-void TeamcityReporter::EnableRetrySupport(bool enabled) {
+void TeamcityReporter::SetRetryLimit(size_t retry_limit) {
   CHECK_EQ(test_suite_stage_, TestSuiteStage::kSuiteStarted);
-  tsm_.TestRetrySupport(enabled);
+  retry_limit_ = retry_limit;
+  tsm_.TestRetrySupport(retry_limit_ != 0);
 }
 
 void TeamcityReporter::OnTestStarted(const TestResult& result) {
@@ -84,6 +109,7 @@ void TeamcityReporter::OnTestResult(const TestResult& result) {
   CHECK_EQ(test_suite_stage_, TestSuiteStage::kTestStarted);
   switch (result.status) {
     case TestResult::TEST_SUCCESS:
+      ClearIgnoredTestFailure(result);
       break;
     case TestResult::TEST_FAILURE:
     case TestResult::TEST_FAILURE_ON_EXIT:
@@ -92,7 +118,11 @@ void TeamcityReporter::OnTestResult(const TestResult& result) {
     case TestResult::TEST_EXCESSIVE_OUTPUT:
     case TestResult::TEST_UNKNOWN:
     case TestResult::TEST_NOT_RUN:
-      tsm_.TestFailed(result.full_name);
+      if (ShouldIgnoreTestFailure(result)) {
+        tsm_.TestIgnored(result.full_name, kPreliminaryFailureIgnoreMessage);
+      } else {
+        tsm_.TestFailed(result.full_name, result.StatusAsString());
+      }
       break;
     case TestResult::TEST_SKIPPED:
       CHECK(false) << "TEST_SKIPPED is unexpected. Please check "
@@ -110,7 +140,7 @@ void TeamcityReporter::OnTestFinished(const TestResult& result) {
   if (result.status == TestResult::TEST_SKIPPED) {
     // This is not a failure nor a success. Mark the test as ignored to not add
     // it into "successful/failed" lists.
-    tsm_.TestIgnored(result.full_name);
+    tsm_.TestIgnored(result.full_name, kTestSkippedIgnoreMessage);
   }
   tsm_.TestFinished(result.full_name, result.elapsed_time);
   test_suite_stage_ = TestSuiteStage::kTestFinished;
@@ -128,9 +158,52 @@ void TeamcityReporter::LogSuiteStarted() {
 
 void TeamcityReporter::LogSuiteFinished() {
   if (test_suite_stage_ != TestSuiteStage::kSuiteFinished) {
+    ReportIgnoredTestFailures();
     tsm_.TestSuiteFinished(suite_name_);
     test_suite_stage_ = TestSuiteStage::kSuiteFinished;
   }
 }
 
+bool TeamcityReporter::ShouldIgnoreTestFailure(const TestResult& result) {
+  CHECK_EQ(test_suite_stage_, TestSuiteStage::kTestStarted);
+  CHECK_NE(result.status, TestResult::TEST_SUCCESS);
+
+  if (ignore_preliminary_failures_ && retry_limit_ > 0) {
+    auto& test_failure = ignored_test_failures_[result.full_name];
+    if (test_failure.attempt < retry_limit_) {
+      // The test has failed, but we're ignoring it for now.
+      ++test_failure.attempt;
+      // Store the result to report it on early exit.
+      test_failure.result = result;
+      return true;
+    } else {
+      // The test is about to be reported. Unset the result to avoid double
+      // reporting on early exit.
+      test_failure.result.reset();
+    }
+  }
+
+  return false;
+}
+
+void TeamcityReporter::ClearIgnoredTestFailure(const TestResult& result) {
+  CHECK_EQ(test_suite_stage_, TestSuiteStage::kTestStarted);
+  CHECK_EQ(result.status, TestResult::TEST_SUCCESS);
+  ignored_test_failures_.erase(result.full_name);
+}
+
+void TeamcityReporter::ReportIgnoredTestFailures() {
+  for (const auto& [test_name, test_failure] : ignored_test_failures_) {
+    if (test_failure.result) {
+      const TestResult& result = *test_failure.result;
+      tsm_.TestStarted(test_name);
+      tsm_.TestFailed(
+          test_name,
+          JoinString({kNotRetriedMessage, result.StatusAsString()}, "\n"),
+          result.output_snippet);
+      tsm_.TestFinished(test_name, result.elapsed_time);
+    }
+  }
+}
+
 }  // namespace base

base/test/launcher/teamcity_reporter.h

@@ -6,16 +6,18 @@
 #ifndef BRAVE_BASE_TEST_LAUNCHER_TEAMCITY_REPORTER_H_
 #define BRAVE_BASE_TEST_LAUNCHER_TEAMCITY_REPORTER_H_
 
+#include <map>
 #include <memory>
+#include <optional>
 #include <ostream>
 #include <string>
+#include <string_view>
 
+#include "base/test/launcher/test_result.h"
 #include "brave/base/test/launcher/teamcity_service_messages.h"
 
 namespace base {
 
-struct TestResult;
-
 // Reports test results to Teamcity using Service Messages.
 class TeamcityReporter {
  public:
@@ -23,14 +25,16 @@ class TeamcityReporter {
   // a command line flag is passed.
   static std::unique_ptr<TeamcityReporter> MaybeCreate();
 
-  TeamcityReporter(std::ostream& ostream, std::string suite_name);
+  TeamcityReporter(std::ostream& ostream,
+                   std::string suite_name,
+                   bool ignore_preliminary_failures);
   TeamcityReporter(const TeamcityReporter&) = delete;
   TeamcityReporter& operator=(const TeamcityReporter&) = delete;
   ~TeamcityReporter();
 
   // Enable or disable retry support on Teamcity. With this option enabled, the
   // successful run of a test will mute its previous failure.
-  void EnableRetrySupport(bool enabled);
+  void SetRetryLimit(size_t retry_limit);
 
   void OnTestStarted(const TestResult& result);
   void OnTestResult(const TestResult& result);
@@ -40,6 +44,11 @@ class TeamcityReporter {
   // do an early exit.
   void OnBrokenTestEarlyExit();
 
+  // Public for tests.
+  static const std::string_view kPreliminaryFailureIgnoreMessage;
+  static const std::string_view kTestSkippedIgnoreMessage;
+  static const std::string_view kNotRetriedMessage;
+
  private:
   enum class TestSuiteStage {
     kNone,
@@ -50,15 +59,46 @@ class TeamcityReporter {
     kSuiteFinished,
   };
 
+  struct TestFailure {
+    TestFailure();
+    TestFailure(const TestFailure&) = delete;
+    TestFailure& operator=(const TestFailure&) = delete;
+    ~TestFailure();
+
+    size_t attempt = 0;
+    std::optional<TestResult> result;
+  };
+
   void LogSuiteStarted();
   void LogSuiteFinished();
 
+  // A test failure can be ignored if it is a preliminary failure which may be
+  // fixed on retry.
+  bool ShouldIgnoreTestFailure(const TestResult& result);
+  // If a test is successful on retry, the previous failure should be cleared to
+  // not report on shutdown.
+  void ClearIgnoredTestFailure(const TestResult& result);
+  // Report ignored test failures. Used on shutdown.
+  void ReportIgnoredTestFailures();
+
   TeamcityServiceMessages tsm_;
   const std::string suite_name_;
 
+  // Skips initial failures when retries are on, reporting only final test
+  // results. Useful for test suites with flaky tests, where flakiness reporting
+  // is not a concern and no fix is intended (e.g. upstream tests).
+  const bool ignore_preliminary_failures_;
+
+  // The number of retries allowed for each test.
+  size_t retry_limit_ = 0;
+
   // The current test suite stage. This is used to ensure that the test
   // callbacks are called in the correct order.
   TestSuiteStage test_suite_stage_ = TestSuiteStage::kNone;
+
+  // Test failures to be reported on early exit if ignore_preliminary_failures_
+  // is enabled.
+  std::map<std::string, TestFailure> ignored_test_failures_;
 };
 
 }  // namespace base

base/test/launcher/teamcity_reporter_unittest.cc

@@ -13,11 +13,13 @@
 
 namespace base {
 
-class TeamcityReporterTest : public testing::Test {
+class TeamcityReporterTest : public testing::TestWithParam<bool> {
  protected:
   void SetUp() override {
-    teamcity_reporter_ =
-        std::make_unique<TeamcityReporter>(mock_ostream_, "my_suite");
+    // Instantiate reporter with enabled/disable preliminary failures reporting.
+    // The reporter should behave the same, because the retry limit is not set.
+    teamcity_reporter_ = std::make_unique<TeamcityReporter>(
+        mock_ostream_, "my_suite", GetParam());
     EXPECT_EQ(GetStr(), "##teamcity[testSuiteStarted name='my_suite']\n");
   }
 
@@ -40,12 +42,15 @@ class TeamcityReporterTest : public testing::Test {
   std::unique_ptr<TeamcityReporter> teamcity_reporter_;
 };
 
-TEST_F(TeamcityReporterTest, EnableRetrySupport) {
-  teamcity_reporter_->EnableRetrySupport(true);
+TEST_P(TeamcityReporterTest, SetRetryLimit) {
+  teamcity_reporter_->SetRetryLimit(1);
   EXPECT_EQ(GetStr(), "##teamcity[testRetrySupport enabled='true']\n");
+
+  teamcity_reporter_->SetRetryLimit(0);
+  EXPECT_EQ(GetStr(), "##teamcity[testRetrySupport enabled='false']\n");
 }
 
-TEST_F(TeamcityReporterTest, TestSuccessful) {
+TEST_P(TeamcityReporterTest, TestSuccessful) {
   TestResult result;
   result.full_name = "TestSuite.TestName";
   result.status = TestResult::TEST_SUCCESS;
@@ -60,7 +65,7 @@ TEST_F(TeamcityReporterTest, TestSuccessful) {
       "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n");
 }
 
-TEST_F(TeamcityReporterTest, TestFailed) {
+TEST_P(TeamcityReporterTest, TestFailed) {
   TestResult result;
   result.full_name = "TestSuite.TestName";
   result.status = TestResult::TEST_FAILURE;
@@ -72,26 +77,29 @@ TEST_F(TeamcityReporterTest, TestFailed) {
       GetStr(),
       "##teamcity[testStarted name='TestSuite.TestName' "
       "captureStandardOutput='true']\n"
-      "##teamcity[testFailed name='TestSuite.TestName']\n"
+      "##teamcity[testFailed name='TestSuite.TestName' message='FAILURE']\n"
       "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n");
 }
 
-TEST_F(TeamcityReporterTest, TestSkipped) {
+TEST_P(TeamcityReporterTest, TestSkipped) {
   TestResult result;
   result.full_name = "TestSuite.TestName";
   result.status = TestResult::TEST_SKIPPED;
   result.elapsed_time = Milliseconds(100);
   teamcity_reporter_->OnTestStarted(result);
   teamcity_reporter_->OnTestFinished(result);
-  EXPECT_EQ(
-      GetStr(),
-      "##teamcity[testStarted name='TestSuite.TestName' "
-      "captureStandardOutput='true']\n"
-      "##teamcity[testIgnored name='TestSuite.TestName']\n"
-      "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n");
+  EXPECT_EQ(GetStr(),
+            "##teamcity[testStarted name='TestSuite.TestName' "
+            "captureStandardOutput='true']\n"
+            "##teamcity[testIgnored name='TestSuite.TestName' "
+            "message='" +
+                std::string(TeamcityReporter::kTestSkippedIgnoreMessage) +
+                "']\n"
+                "##teamcity[testFinished name='TestSuite.TestName' "
+                "duration='100']\n");
 }
 
-TEST_F(TeamcityReporterTest, OnBrokenTestEarlyExit) {
+TEST_P(TeamcityReporterTest, OnBrokenTestEarlyExit) {
   teamcity_reporter_->OnBrokenTestEarlyExit();
   EXPECT_EQ(GetStr(), "##teamcity[testSuiteFinished name='my_suite']\n");
 
@@ -102,7 +110,7 @@ TEST_F(TeamcityReporterTest, OnBrokenTestEarlyExit) {
   EXPECT_EQ(GetStr(), "");
 }
 
-TEST_F(TeamcityReporterTest, MissingResultOnSuccess) {
+TEST_P(TeamcityReporterTest, MissingResultOnSuccess) {
   TestResult result;
   result.full_name = "TestSuite.TestName";
   result.status = TestResult::TEST_SUCCESS;
@@ -111,7 +119,7 @@ TEST_F(TeamcityReporterTest, MissingResultOnSuccess) {
   EXPECT_DEATH_IF_SUPPORTED(teamcity_reporter_->OnTestFinished(result), "");
 }
 
-TEST_F(TeamcityReporterTest, MissingResultOnFailure) {
+TEST_P(TeamcityReporterTest, MissingResultOnFailure) {
   TestResult result;
   result.full_name = "TestSuite.TestName";
   result.status = TestResult::TEST_FAILURE;
@@ -120,7 +128,7 @@ TEST_F(TeamcityReporterTest, MissingResultOnFailure) {
   EXPECT_DEATH_IF_SUPPORTED(teamcity_reporter_->OnTestFinished(result), "");
 }
 
-TEST_F(TeamcityReporterTest, UnexpectedResultOnSkipped) {
+TEST_P(TeamcityReporterTest, UnexpectedResultOnSkipped) {
   TestResult result;
   result.full_name = "TestSuite.TestName";
   result.status = TestResult::TEST_SKIPPED;
@@ -129,7 +137,7 @@ TEST_F(TeamcityReporterTest, UnexpectedResultOnSkipped) {
   EXPECT_DEATH_IF_SUPPORTED(teamcity_reporter_->OnTestResult(result), "");
 }
 
-TEST_F(TeamcityReporterTest, MissingStart) {
+TEST_P(TeamcityReporterTest, MissingStart) {
   TestResult result;
   result.full_name = "TestSuite.TestName";
   result.status = TestResult::TEST_SUCCESS;
@@ -137,7 +145,7 @@ TEST_F(TeamcityReporterTest, MissingStart) {
   EXPECT_DEATH_IF_SUPPORTED(teamcity_reporter_->OnTestFinished(result), "");
 }
 
-TEST_F(TeamcityReporterTest, NoReportingAfterEarlyExit) {
+TEST_P(TeamcityReporterTest, NoReportingAfterEarlyExit) {
   teamcity_reporter_->OnBrokenTestEarlyExit();
   ClearStr();
 
@@ -149,4 +157,164 @@ TEST_F(TeamcityReporterTest, NoReportingAfterEarlyExit) {
   teamcity_reporter_.reset();
 }
 
+INSTANTIATE_TEST_SUITE_P(, TeamcityReporterTest, testing::Bool());
+
+class TeamcityReporterIgnorePreliminaryFailuresTest
+    : public TeamcityReporterTest {
+ protected:
+  void SetUp() override {
+    teamcity_reporter_ =
+        std::make_unique<TeamcityReporter>(mock_ostream_, "my_suite", true);
+    teamcity_reporter_->SetRetryLimit(1);
+    EXPECT_EQ(GetStr(),
+              "##teamcity[testSuiteStarted name='my_suite']\n"
+              "##teamcity[testRetrySupport enabled='true']\n");
+  }
+};
+
+TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, TestSuccessful) {
+  TestResult result;
+  result.full_name = "TestSuite.TestName";
+  result.status = TestResult::TEST_SUCCESS;
+  result.elapsed_time = Milliseconds(100);
+  teamcity_reporter_->OnTestStarted(result);
+  teamcity_reporter_->OnTestResult(result);
+  teamcity_reporter_->OnTestFinished(result);
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n");
+}
+
+TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, TestFailedOnRetry) {
+  TestResult result;
+  result.full_name = "TestSuite.TestName";
+  result.status = TestResult::TEST_FAILURE;
+  result.elapsed_time = Milliseconds(100);
+  teamcity_reporter_->OnTestStarted(result);
+  teamcity_reporter_->OnTestResult(result);
+  teamcity_reporter_->OnTestFinished(result);
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testIgnored name='TestSuite.TestName' "
+      "message='" +
+          std::string(TeamcityReporter::kPreliminaryFailureIgnoreMessage) +
+          "']\n"
+          "##teamcity[testFinished name='TestSuite.TestName' "
+          "duration='100']\n");
+
+  teamcity_reporter_->OnTestStarted(result);
+  teamcity_reporter_->OnTestResult(result);
+  teamcity_reporter_->OnTestFinished(result);
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testFailed name='TestSuite.TestName' message='FAILURE']\n"
+      "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n");
+}
+
+TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, TestSuccessfulOnRetry) {
+  TestResult result;
+  result.full_name = "TestSuite.TestName";
+  result.status = TestResult::TEST_FAILURE;
+  result.elapsed_time = Milliseconds(100);
+  result.output_snippet = "output";
+  teamcity_reporter_->OnTestStarted(result);
+  teamcity_reporter_->OnTestResult(result);
+  teamcity_reporter_->OnTestFinished(result);
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testIgnored name='TestSuite.TestName' "
+      "message='" +
+          std::string(TeamcityReporter::kPreliminaryFailureIgnoreMessage) +
+          "']\n"
+          "##teamcity[testFinished name='TestSuite.TestName' "
+          "duration='100']\n");
+
+  result.status = TestResult::TEST_SUCCESS;
+  teamcity_reporter_->OnTestStarted(result);
+  teamcity_reporter_->OnTestResult(result);
+  teamcity_reporter_->OnTestFinished(result);
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n");
+}
+
+TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, OnBrokenTestEarlyExit) {
+  TestResult result;
+  result.full_name = "TestSuite.TestName";
+  result.status = TestResult::TEST_FAILURE;
+  result.elapsed_time = Milliseconds(100);
+  result.output_snippet = "output";
+  teamcity_reporter_->OnTestStarted(result);
+  teamcity_reporter_->OnTestResult(result);
+  teamcity_reporter_->OnTestFinished(result);
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testIgnored name='TestSuite.TestName' "
+      "message='" +
+          std::string(TeamcityReporter::kPreliminaryFailureIgnoreMessage) +
+          "']\n"
+          "##teamcity[testFinished name='TestSuite.TestName' "
+          "duration='100']\n");
+
+  teamcity_reporter_->OnBrokenTestEarlyExit();
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testFailed name='TestSuite.TestName' "
+      "message='" +
+          std::string(TeamcityReporter::kNotRetriedMessage) +
+          "|nFAILURE' details='output']\n"
+          "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n"
+          "##teamcity[testSuiteFinished name='my_suite']\n");
+
+  teamcity_reporter_.reset();
+  EXPECT_EQ(GetStr(), "");
+}
+
+TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, Shutdown) {
+  TestResult result;
+  result.full_name = "TestSuite.TestName";
+  result.status = TestResult::TEST_FAILURE;
+  result.elapsed_time = Milliseconds(100);
+  result.output_snippet = "output";
+  teamcity_reporter_->OnTestStarted(result);
+  teamcity_reporter_->OnTestResult(result);
+  teamcity_reporter_->OnTestFinished(result);
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testIgnored name='TestSuite.TestName' "
+      "message='" +
+          std::string(TeamcityReporter::kPreliminaryFailureIgnoreMessage) +
+          "']\n"
+          "##teamcity[testFinished name='TestSuite.TestName' "
+          "duration='100']\n");
+
+  teamcity_reporter_.reset();
+  EXPECT_EQ(
+      GetStr(),
+      "##teamcity[testStarted name='TestSuite.TestName' "
+      "captureStandardOutput='true']\n"
+      "##teamcity[testFailed name='TestSuite.TestName' "
+      "message='" +
+          std::string(TeamcityReporter::kNotRetriedMessage) +
+          "|nFAILURE' details='output']\n"
+          "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n"
+          "##teamcity[testSuiteFinished name='my_suite']\n");
+}
+
 }  // namespace base

build/commands/lib/test.js

@@ -2,6 +2,7 @@ const fs = require('fs-extra')
 const path = require('path')
 
 const config = require('../lib/config')
+const Log = require('../lib/logging')
 const util = require('../lib/util')
 const assert = require('assert')
 
@@ -129,6 +130,9 @@ const runTests = (passthroughArgs, suite, buildConfig, options) => {
     let filterFilePaths = getApplicableFilters(suite)
     if (filterFilePaths.length > 0)
       braveArgs.push(`--test-launcher-filter-file="${filterFilePaths.join(';')}"`)
+    if (config.isTeamcity) {
+      braveArgs.push('--test-launcher-teamcity-reporter-ignore-preliminary-failures')
+    }
   }
 
   if (
@@ -137,7 +141,7 @@ const runTests = (passthroughArgs, suite, buildConfig, options) => {
     config.targetOS !== 'android' &&
     config.targetOS !== 'ios'
   ) {
-    runChromiumTestLauncherTeamcityReporterIntegrationTest()
+    runChromiumTestLauncherTeamcityReporterIntegrationTests()
   }
 
   if (config.targetOS === 'ios') {
@@ -186,54 +190,139 @@ const runTests = (passthroughArgs, suite, buildConfig, options) => {
   }
 }
 
-const runChromiumTestLauncherTeamcityReporterIntegrationTest = () => {
-  const args = [
-    "--gtest_filter=DISABLED_TeamcityReporterIntegration*",
-    "--gtest_also_run_disabled_tests",
-  ]
+const runChromiumTestLauncherTeamcityReporterIntegrationTests = () => {
+  const generalTestCase = {
+    args: [
+      '--test-launcher-bot-mode',
+      '--gtest_filter=DISABLED_TeamcityReporterIntegration*',
+      '--gtest_also_run_disabled_tests',
+      // Enable retry limit explicitly, because it's set to 0 when
+      // --gtest_filter is passed.
+      '--test-launcher-retry-limit=1',
+    ],
+
+    expectedLines: [
+      "##teamcity[testSuiteStarted name='brave_unit_tests']",
+      "##teamcity[testRetrySupport enabled='true']",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Success'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Success'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testSuiteFinished name='brave_unit_tests']"
+    ]
+  }
+
+  const ignorePreliminaryFailuresTestCase = {
+    args: [
+      ...generalTestCase.args,
+      '--test-launcher-teamcity-reporter-ignore-preliminary-failures'
+    ],
+
+    expectedLines: [
+      "##teamcity[testSuiteStarted name='brave_unit_tests']",
+      "##teamcity[testRetrySupport enabled='true']",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Success'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Success'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'",
+      "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped'",
+      "##teamcity[testSuiteFinished name='brave_unit_tests']"
+    ]
+  }
 
   const runOptions = config.defaultOptions
   runOptions.stdio = 'pipe'
   runOptions.continueOnFail = true
-  const prog = util.run(path.join(config.outputDir, 'brave_unit_tests'), args, runOptions)
 
-  const expectedOutput = [
-    "##teamcity[testSuiteStarted name='brave_unit_tests']",
-    "##teamcity[testRetrySupport enabled='false']",
-    "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Success' captureStandardOutput='true']",
-    "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Success' duration='",
-    "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure' captureStandardOutput='true']",
-    "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.Failure']",
-    "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure' duration='",
-    "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure' captureStandardOutput='true']",
-    "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure']",
-    "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure' duration='",
-    "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped' captureStandardOutput='true']",
-    "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.Skipped']",
-    "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped' duration='",
-    "##teamcity[testSuiteFinished name='brave_unit_tests']",
-  ]
-
-  const stdoutLines = prog.stdout.toString().split('\n');
-  for (const line of stdoutLines) {
-    if (line.startsWith(expectedOutput[0])) {
-      expectedOutput.splice(0, 1)
-      if (expectedOutput.length == 0) {
-        break
-      }
-    }
+  for (const testCase of [generalTestCase, ignorePreliminaryFailuresTestCase]) {
+    const prog = util.run(
+      path.join(config.outputDir, 'brave_unit_tests'),
+      testCase.args,
+      runOptions
+    )
+    const outputLines = prog.stdout.toString().split('\n')
+    checkTeamcityReporterOutput(outputLines, testCase.expectedLines)
   }
+}
+
+const checkTeamcityReporterOutput = (outputLines, expectedTeamcityLines) => {
+  const outputTeamcityLines = outputLines.filter((line) =>
+    line.startsWith('##teamcity')
+  )
+
+  const isMatched =
+    outputTeamcityLines.length === expectedTeamcityLines.length &&
+    expectedTeamcityLines.every((expectedLine, index) => {
+      const outputLine = outputTeamcityLines[index]
+      return outputLine.startsWith(expectedLine)
+    })
 
-  const isMatched = expectedOutput.length == 0;
   if (!isMatched) {
+    const notMatchedOutputLines = outputTeamcityLines.filter(
+      (outputLine, index) => {
+        const expectedLine = expectedTeamcityLines[index]
+        return !outputLine.startsWith(expectedLine)
+      }
+    )
+
+    const notMatchedExpectedLines = expectedTeamcityLines.filter(
+      (expectedLine, index) => {
+        const outputLine = outputTeamcityLines[index]
+        return !outputLine?.startsWith(expectedLine)
+      }
+    )
+
+    Log.error(
+      'TeamcityReporter output test failed, output ##teamcity lines do not match expected lines (note ##teamcity was replaced with %%teamcity):'
+    )
     console.error(
-      prog.stdout.toString().replace(/##teamcity/gm, '%%teamcity') +
-      '\nChromiumTestLauncherTeamcityReporterIntegration test failed, the line was not found in the output (note: ##teamcity was replaced with %%teamcity):\n' +
-      expectedOutput[0].replace(/##teamcity/gm, '%%teamcity')
+      [
+        '\nNot matched output lines:',
+        ...notMatchedOutputLines,
+        '\nNot matched expected lines:',
+        ...notMatchedExpectedLines,
+        '\nTest output lines:',
+        ...outputTeamcityLines,
+        '\nExpected lines:',
+        ...expectedTeamcityLines,
+        '\nFull test output:',
+        ...outputLines
+      ]
+        .join('\n')
+        .replace(/##teamcity/gm, '%%teamcity')
     )
     process.exit(1)
   } else {
-    console.log('ChromiumTestLauncherTeamcityReporterIntegration test passed')
+    console.log('TeamcityReporter output test passed')
   }
 }
 

chromium_src/base/test/launcher/test_launcher.cc

@@ -54,7 +54,7 @@ void TestLauncher::CreateAndStartThreadPool(size_t num_parallel_jobs) {
   // `retry_limit_` can be overridden by command line. Read its value when all
   // command line flags are parsed.
   if (teamcity_reporter_) {
-    teamcity_reporter_->EnableRetrySupport(retry_limit_ != 0);
+    teamcity_reporter_->SetRetryLimit(retry_limit_);
   }
 
   TestLauncher_ChromiumImpl::CreateAndStartThreadPool(num_parallel_jobs);