diff --git a/base/test/launcher/teamcity_reporter.cc b/base/test/launcher/teamcity_reporter.cc index e2ea2c2b18e..ac2c77b6fad 100644 --- a/base/test/launcher/teamcity_reporter.cc +++ b/base/test/launcher/teamcity_reporter.cc @@ -13,6 +13,7 @@ #include "base/environment.h" #include "base/files/file_path.h" #include "base/path_service.h" +#include "base/strings/string_util.h" #include "base/test/launcher/test_result.h" namespace base { @@ -28,6 +29,11 @@ constexpr char kTestLauncherEnableTeamcityReporter[] = constexpr char kTestLauncherDisableTeamcityReporter[] = "test-launcher-disable-teamcity-reporter"; +// This switch enables the TeamcityReporter to ignore preliminary test failures +// when test retries are enabled, reporting only the final result of each test. +constexpr char kTestLauncherTeamcityReporterIgnorePreliminaryFailures[] = + "test-launcher-teamcity-reporter-ignore-preliminary-failures"; + // Returns the name of the current executable, excluding the extension. std::string GetExecutableName() { return PathService::CheckedGet(FILE_EXE) @@ -38,6 +44,18 @@ std::string GetExecutableName() { } // namespace +constexpr std::string_view TeamcityReporter::kPreliminaryFailureIgnoreMessage = + "Failure ignored, expecting a retry"; + +constexpr std::string_view TeamcityReporter::kTestSkippedIgnoreMessage = + "Skipped, possibly because of a previous failure"; + +constexpr std::string_view TeamcityReporter::kNotRetriedMessage = + "NOT_RETRIED (suite early exit)"; + +TeamcityReporter::TestFailure::TestFailure() = default; +TeamcityReporter::TestFailure::~TestFailure() = default; + // static std::unique_ptr TeamcityReporter::MaybeCreate() { const auto environment = Environment::Create(); @@ -51,15 +69,21 @@ std::unique_ptr TeamcityReporter::MaybeCreate() { command_line->HasSwitch(kTestLauncherDisableTeamcityReporter); if (should_enable && !should_disable) { - return std::make_unique(std::cout, GetExecutableName()); + const bool ignore_preliminary_failures = command_line->HasSwitch( + kTestLauncherTeamcityReporterIgnorePreliminaryFailures); + return std::make_unique(std::cout, GetExecutableName(), + ignore_preliminary_failures); } return nullptr; } TeamcityReporter::TeamcityReporter(std::ostream& ostream, - std::string suite_name) - : tsm_(ostream), suite_name_(std::move(suite_name)) { + std::string suite_name, + bool ignore_preliminary_failures) + : tsm_(ostream), + suite_name_(std::move(suite_name)), + ignore_preliminary_failures_(ignore_preliminary_failures) { LogSuiteStarted(); } @@ -67,9 +91,10 @@ TeamcityReporter::~TeamcityReporter() { LogSuiteFinished(); } -void TeamcityReporter::EnableRetrySupport(bool enabled) { +void TeamcityReporter::SetRetryLimit(size_t retry_limit) { CHECK_EQ(test_suite_stage_, TestSuiteStage::kSuiteStarted); - tsm_.TestRetrySupport(enabled); + retry_limit_ = retry_limit; + tsm_.TestRetrySupport(retry_limit_ != 0); } void TeamcityReporter::OnTestStarted(const TestResult& result) { @@ -84,6 +109,7 @@ void TeamcityReporter::OnTestResult(const TestResult& result) { CHECK_EQ(test_suite_stage_, TestSuiteStage::kTestStarted); switch (result.status) { case TestResult::TEST_SUCCESS: + ClearIgnoredTestFailure(result); break; case TestResult::TEST_FAILURE: case TestResult::TEST_FAILURE_ON_EXIT: @@ -92,7 +118,11 @@ void TeamcityReporter::OnTestResult(const TestResult& result) { case TestResult::TEST_EXCESSIVE_OUTPUT: case TestResult::TEST_UNKNOWN: case TestResult::TEST_NOT_RUN: - tsm_.TestFailed(result.full_name); + if (ShouldIgnoreTestFailure(result)) { + tsm_.TestIgnored(result.full_name, kPreliminaryFailureIgnoreMessage); + } else { + tsm_.TestFailed(result.full_name, result.StatusAsString()); + } break; case TestResult::TEST_SKIPPED: CHECK(false) << "TEST_SKIPPED is unexpected. Please check " @@ -110,7 +140,7 @@ void TeamcityReporter::OnTestFinished(const TestResult& result) { if (result.status == TestResult::TEST_SKIPPED) { // This is not a failure nor a success. Mark the test as ignored to not add // it into "successful/failed" lists. - tsm_.TestIgnored(result.full_name); + tsm_.TestIgnored(result.full_name, kTestSkippedIgnoreMessage); } tsm_.TestFinished(result.full_name, result.elapsed_time); test_suite_stage_ = TestSuiteStage::kTestFinished; @@ -128,9 +158,52 @@ void TeamcityReporter::LogSuiteStarted() { void TeamcityReporter::LogSuiteFinished() { if (test_suite_stage_ != TestSuiteStage::kSuiteFinished) { + ReportIgnoredTestFailures(); tsm_.TestSuiteFinished(suite_name_); test_suite_stage_ = TestSuiteStage::kSuiteFinished; } } +bool TeamcityReporter::ShouldIgnoreTestFailure(const TestResult& result) { + CHECK_EQ(test_suite_stage_, TestSuiteStage::kTestStarted); + CHECK_NE(result.status, TestResult::TEST_SUCCESS); + + if (ignore_preliminary_failures_ && retry_limit_ > 0) { + auto& test_failure = ignored_test_failures_[result.full_name]; + if (test_failure.attempt < retry_limit_) { + // The test has failed, but we're ignoring it for now. + ++test_failure.attempt; + // Store the result to report it on early exit. + test_failure.result = result; + return true; + } else { + // The test is about to be reported. Unset the result to avoid double + // reporting on early exit. + test_failure.result.reset(); + } + } + + return false; +} + +void TeamcityReporter::ClearIgnoredTestFailure(const TestResult& result) { + CHECK_EQ(test_suite_stage_, TestSuiteStage::kTestStarted); + CHECK_EQ(result.status, TestResult::TEST_SUCCESS); + ignored_test_failures_.erase(result.full_name); +} + +void TeamcityReporter::ReportIgnoredTestFailures() { + for (const auto& [test_name, test_failure] : ignored_test_failures_) { + if (test_failure.result) { + const TestResult& result = *test_failure.result; + tsm_.TestStarted(test_name); + tsm_.TestFailed( + test_name, + JoinString({kNotRetriedMessage, result.StatusAsString()}, "\n"), + result.output_snippet); + tsm_.TestFinished(test_name, result.elapsed_time); + } + } +} + } // namespace base diff --git a/base/test/launcher/teamcity_reporter.h b/base/test/launcher/teamcity_reporter.h index 594d6adf0a2..fd216e26ab2 100644 --- a/base/test/launcher/teamcity_reporter.h +++ b/base/test/launcher/teamcity_reporter.h @@ -6,16 +6,18 @@ #ifndef BRAVE_BASE_TEST_LAUNCHER_TEAMCITY_REPORTER_H_ #define BRAVE_BASE_TEST_LAUNCHER_TEAMCITY_REPORTER_H_ +#include #include +#include #include #include +#include +#include "base/test/launcher/test_result.h" #include "brave/base/test/launcher/teamcity_service_messages.h" namespace base { -struct TestResult; - // Reports test results to Teamcity using Service Messages. class TeamcityReporter { public: @@ -23,14 +25,16 @@ class TeamcityReporter { // a command line flag is passed. static std::unique_ptr MaybeCreate(); - TeamcityReporter(std::ostream& ostream, std::string suite_name); + TeamcityReporter(std::ostream& ostream, + std::string suite_name, + bool ignore_preliminary_failures); TeamcityReporter(const TeamcityReporter&) = delete; TeamcityReporter& operator=(const TeamcityReporter&) = delete; ~TeamcityReporter(); // Enable or disable retry support on Teamcity. With this option enabled, the // successful run of a test will mute its previous failure. - void EnableRetrySupport(bool enabled); + void SetRetryLimit(size_t retry_limit); void OnTestStarted(const TestResult& result); void OnTestResult(const TestResult& result); @@ -40,6 +44,11 @@ class TeamcityReporter { // do an early exit. void OnBrokenTestEarlyExit(); + // Public for tests. + static const std::string_view kPreliminaryFailureIgnoreMessage; + static const std::string_view kTestSkippedIgnoreMessage; + static const std::string_view kNotRetriedMessage; + private: enum class TestSuiteStage { kNone, @@ -50,15 +59,46 @@ class TeamcityReporter { kSuiteFinished, }; + struct TestFailure { + TestFailure(); + TestFailure(const TestFailure&) = delete; + TestFailure& operator=(const TestFailure&) = delete; + ~TestFailure(); + + size_t attempt = 0; + std::optional result; + }; + void LogSuiteStarted(); void LogSuiteFinished(); + // A test failure can be ignored if it is a preliminary failure which may be + // fixed on retry. + bool ShouldIgnoreTestFailure(const TestResult& result); + // If a test is successful on retry, the previous failure should be cleared to + // not report on shutdown. + void ClearIgnoredTestFailure(const TestResult& result); + // Report ignored test failures. Used on shutdown. + void ReportIgnoredTestFailures(); + TeamcityServiceMessages tsm_; const std::string suite_name_; + // Skips initial failures when retries are on, reporting only final test + // results. Useful for test suites with flaky tests, where flakiness reporting + // is not a concern and no fix is intended (e.g. upstream tests). + const bool ignore_preliminary_failures_; + + // The number of retries allowed for each test. + size_t retry_limit_ = 0; + // The current test suite stage. This is used to ensure that the test // callbacks are called in the correct order. TestSuiteStage test_suite_stage_ = TestSuiteStage::kNone; + + // Test failures to be reported on early exit if ignore_preliminary_failures_ + // is enabled. + std::map ignored_test_failures_; }; } // namespace base diff --git a/base/test/launcher/teamcity_reporter_unittest.cc b/base/test/launcher/teamcity_reporter_unittest.cc index 0eead067f99..1a31e641c0b 100644 --- a/base/test/launcher/teamcity_reporter_unittest.cc +++ b/base/test/launcher/teamcity_reporter_unittest.cc @@ -13,11 +13,13 @@ namespace base { -class TeamcityReporterTest : public testing::Test { +class TeamcityReporterTest : public testing::TestWithParam { protected: void SetUp() override { - teamcity_reporter_ = - std::make_unique(mock_ostream_, "my_suite"); + // Instantiate reporter with enabled/disable preliminary failures reporting. + // The reporter should behave the same, because the retry limit is not set. + teamcity_reporter_ = std::make_unique( + mock_ostream_, "my_suite", GetParam()); EXPECT_EQ(GetStr(), "##teamcity[testSuiteStarted name='my_suite']\n"); } @@ -40,12 +42,15 @@ class TeamcityReporterTest : public testing::Test { std::unique_ptr teamcity_reporter_; }; -TEST_F(TeamcityReporterTest, EnableRetrySupport) { - teamcity_reporter_->EnableRetrySupport(true); +TEST_P(TeamcityReporterTest, SetRetryLimit) { + teamcity_reporter_->SetRetryLimit(1); EXPECT_EQ(GetStr(), "##teamcity[testRetrySupport enabled='true']\n"); + + teamcity_reporter_->SetRetryLimit(0); + EXPECT_EQ(GetStr(), "##teamcity[testRetrySupport enabled='false']\n"); } -TEST_F(TeamcityReporterTest, TestSuccessful) { +TEST_P(TeamcityReporterTest, TestSuccessful) { TestResult result; result.full_name = "TestSuite.TestName"; result.status = TestResult::TEST_SUCCESS; @@ -60,7 +65,7 @@ TEST_F(TeamcityReporterTest, TestSuccessful) { "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n"); } -TEST_F(TeamcityReporterTest, TestFailed) { +TEST_P(TeamcityReporterTest, TestFailed) { TestResult result; result.full_name = "TestSuite.TestName"; result.status = TestResult::TEST_FAILURE; @@ -72,26 +77,29 @@ TEST_F(TeamcityReporterTest, TestFailed) { GetStr(), "##teamcity[testStarted name='TestSuite.TestName' " "captureStandardOutput='true']\n" - "##teamcity[testFailed name='TestSuite.TestName']\n" + "##teamcity[testFailed name='TestSuite.TestName' message='FAILURE']\n" "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n"); } -TEST_F(TeamcityReporterTest, TestSkipped) { +TEST_P(TeamcityReporterTest, TestSkipped) { TestResult result; result.full_name = "TestSuite.TestName"; result.status = TestResult::TEST_SKIPPED; result.elapsed_time = Milliseconds(100); teamcity_reporter_->OnTestStarted(result); teamcity_reporter_->OnTestFinished(result); - EXPECT_EQ( - GetStr(), - "##teamcity[testStarted name='TestSuite.TestName' " - "captureStandardOutput='true']\n" - "##teamcity[testIgnored name='TestSuite.TestName']\n" - "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n"); + EXPECT_EQ(GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testIgnored name='TestSuite.TestName' " + "message='" + + std::string(TeamcityReporter::kTestSkippedIgnoreMessage) + + "']\n" + "##teamcity[testFinished name='TestSuite.TestName' " + "duration='100']\n"); } -TEST_F(TeamcityReporterTest, OnBrokenTestEarlyExit) { +TEST_P(TeamcityReporterTest, OnBrokenTestEarlyExit) { teamcity_reporter_->OnBrokenTestEarlyExit(); EXPECT_EQ(GetStr(), "##teamcity[testSuiteFinished name='my_suite']\n"); @@ -102,7 +110,7 @@ TEST_F(TeamcityReporterTest, OnBrokenTestEarlyExit) { EXPECT_EQ(GetStr(), ""); } -TEST_F(TeamcityReporterTest, MissingResultOnSuccess) { +TEST_P(TeamcityReporterTest, MissingResultOnSuccess) { TestResult result; result.full_name = "TestSuite.TestName"; result.status = TestResult::TEST_SUCCESS; @@ -111,7 +119,7 @@ TEST_F(TeamcityReporterTest, MissingResultOnSuccess) { EXPECT_DEATH_IF_SUPPORTED(teamcity_reporter_->OnTestFinished(result), ""); } -TEST_F(TeamcityReporterTest, MissingResultOnFailure) { +TEST_P(TeamcityReporterTest, MissingResultOnFailure) { TestResult result; result.full_name = "TestSuite.TestName"; result.status = TestResult::TEST_FAILURE; @@ -120,7 +128,7 @@ TEST_F(TeamcityReporterTest, MissingResultOnFailure) { EXPECT_DEATH_IF_SUPPORTED(teamcity_reporter_->OnTestFinished(result), ""); } -TEST_F(TeamcityReporterTest, UnexpectedResultOnSkipped) { +TEST_P(TeamcityReporterTest, UnexpectedResultOnSkipped) { TestResult result; result.full_name = "TestSuite.TestName"; result.status = TestResult::TEST_SKIPPED; @@ -129,7 +137,7 @@ TEST_F(TeamcityReporterTest, UnexpectedResultOnSkipped) { EXPECT_DEATH_IF_SUPPORTED(teamcity_reporter_->OnTestResult(result), ""); } -TEST_F(TeamcityReporterTest, MissingStart) { +TEST_P(TeamcityReporterTest, MissingStart) { TestResult result; result.full_name = "TestSuite.TestName"; result.status = TestResult::TEST_SUCCESS; @@ -137,7 +145,7 @@ TEST_F(TeamcityReporterTest, MissingStart) { EXPECT_DEATH_IF_SUPPORTED(teamcity_reporter_->OnTestFinished(result), ""); } -TEST_F(TeamcityReporterTest, NoReportingAfterEarlyExit) { +TEST_P(TeamcityReporterTest, NoReportingAfterEarlyExit) { teamcity_reporter_->OnBrokenTestEarlyExit(); ClearStr(); @@ -149,4 +157,164 @@ TEST_F(TeamcityReporterTest, NoReportingAfterEarlyExit) { teamcity_reporter_.reset(); } +INSTANTIATE_TEST_SUITE_P(, TeamcityReporterTest, testing::Bool()); + +class TeamcityReporterIgnorePreliminaryFailuresTest + : public TeamcityReporterTest { + protected: + void SetUp() override { + teamcity_reporter_ = + std::make_unique(mock_ostream_, "my_suite", true); + teamcity_reporter_->SetRetryLimit(1); + EXPECT_EQ(GetStr(), + "##teamcity[testSuiteStarted name='my_suite']\n" + "##teamcity[testRetrySupport enabled='true']\n"); + } +}; + +TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, TestSuccessful) { + TestResult result; + result.full_name = "TestSuite.TestName"; + result.status = TestResult::TEST_SUCCESS; + result.elapsed_time = Milliseconds(100); + teamcity_reporter_->OnTestStarted(result); + teamcity_reporter_->OnTestResult(result); + teamcity_reporter_->OnTestFinished(result); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n"); +} + +TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, TestFailedOnRetry) { + TestResult result; + result.full_name = "TestSuite.TestName"; + result.status = TestResult::TEST_FAILURE; + result.elapsed_time = Milliseconds(100); + teamcity_reporter_->OnTestStarted(result); + teamcity_reporter_->OnTestResult(result); + teamcity_reporter_->OnTestFinished(result); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testIgnored name='TestSuite.TestName' " + "message='" + + std::string(TeamcityReporter::kPreliminaryFailureIgnoreMessage) + + "']\n" + "##teamcity[testFinished name='TestSuite.TestName' " + "duration='100']\n"); + + teamcity_reporter_->OnTestStarted(result); + teamcity_reporter_->OnTestResult(result); + teamcity_reporter_->OnTestFinished(result); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testFailed name='TestSuite.TestName' message='FAILURE']\n" + "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n"); +} + +TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, TestSuccessfulOnRetry) { + TestResult result; + result.full_name = "TestSuite.TestName"; + result.status = TestResult::TEST_FAILURE; + result.elapsed_time = Milliseconds(100); + result.output_snippet = "output"; + teamcity_reporter_->OnTestStarted(result); + teamcity_reporter_->OnTestResult(result); + teamcity_reporter_->OnTestFinished(result); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testIgnored name='TestSuite.TestName' " + "message='" + + std::string(TeamcityReporter::kPreliminaryFailureIgnoreMessage) + + "']\n" + "##teamcity[testFinished name='TestSuite.TestName' " + "duration='100']\n"); + + result.status = TestResult::TEST_SUCCESS; + teamcity_reporter_->OnTestStarted(result); + teamcity_reporter_->OnTestResult(result); + teamcity_reporter_->OnTestFinished(result); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n"); +} + +TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, OnBrokenTestEarlyExit) { + TestResult result; + result.full_name = "TestSuite.TestName"; + result.status = TestResult::TEST_FAILURE; + result.elapsed_time = Milliseconds(100); + result.output_snippet = "output"; + teamcity_reporter_->OnTestStarted(result); + teamcity_reporter_->OnTestResult(result); + teamcity_reporter_->OnTestFinished(result); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testIgnored name='TestSuite.TestName' " + "message='" + + std::string(TeamcityReporter::kPreliminaryFailureIgnoreMessage) + + "']\n" + "##teamcity[testFinished name='TestSuite.TestName' " + "duration='100']\n"); + + teamcity_reporter_->OnBrokenTestEarlyExit(); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testFailed name='TestSuite.TestName' " + "message='" + + std::string(TeamcityReporter::kNotRetriedMessage) + + "|nFAILURE' details='output']\n" + "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n" + "##teamcity[testSuiteFinished name='my_suite']\n"); + + teamcity_reporter_.reset(); + EXPECT_EQ(GetStr(), ""); +} + +TEST_F(TeamcityReporterIgnorePreliminaryFailuresTest, Shutdown) { + TestResult result; + result.full_name = "TestSuite.TestName"; + result.status = TestResult::TEST_FAILURE; + result.elapsed_time = Milliseconds(100); + result.output_snippet = "output"; + teamcity_reporter_->OnTestStarted(result); + teamcity_reporter_->OnTestResult(result); + teamcity_reporter_->OnTestFinished(result); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testIgnored name='TestSuite.TestName' " + "message='" + + std::string(TeamcityReporter::kPreliminaryFailureIgnoreMessage) + + "']\n" + "##teamcity[testFinished name='TestSuite.TestName' " + "duration='100']\n"); + + teamcity_reporter_.reset(); + EXPECT_EQ( + GetStr(), + "##teamcity[testStarted name='TestSuite.TestName' " + "captureStandardOutput='true']\n" + "##teamcity[testFailed name='TestSuite.TestName' " + "message='" + + std::string(TeamcityReporter::kNotRetriedMessage) + + "|nFAILURE' details='output']\n" + "##teamcity[testFinished name='TestSuite.TestName' duration='100']\n" + "##teamcity[testSuiteFinished name='my_suite']\n"); +} + } // namespace base diff --git a/build/commands/lib/test.js b/build/commands/lib/test.js index 17bab943cc7..030dcf479fc 100644 --- a/build/commands/lib/test.js +++ b/build/commands/lib/test.js @@ -2,6 +2,7 @@ const fs = require('fs-extra') const path = require('path') const config = require('../lib/config') +const Log = require('../lib/logging') const util = require('../lib/util') const assert = require('assert') @@ -129,6 +130,9 @@ const runTests = (passthroughArgs, suite, buildConfig, options) => { let filterFilePaths = getApplicableFilters(suite) if (filterFilePaths.length > 0) braveArgs.push(`--test-launcher-filter-file="${filterFilePaths.join(';')}"`) + if (config.isTeamcity) { + braveArgs.push('--test-launcher-teamcity-reporter-ignore-preliminary-failures') + } } if ( @@ -137,7 +141,7 @@ const runTests = (passthroughArgs, suite, buildConfig, options) => { config.targetOS !== 'android' && config.targetOS !== 'ios' ) { - runChromiumTestLauncherTeamcityReporterIntegrationTest() + runChromiumTestLauncherTeamcityReporterIntegrationTests() } if (config.targetOS === 'ios') { @@ -186,54 +190,139 @@ const runTests = (passthroughArgs, suite, buildConfig, options) => { } } -const runChromiumTestLauncherTeamcityReporterIntegrationTest = () => { - const args = [ - "--gtest_filter=DISABLED_TeamcityReporterIntegration*", - "--gtest_also_run_disabled_tests", - ] +const runChromiumTestLauncherTeamcityReporterIntegrationTests = () => { + const generalTestCase = { + args: [ + '--test-launcher-bot-mode', + '--gtest_filter=DISABLED_TeamcityReporterIntegration*', + '--gtest_also_run_disabled_tests', + // Enable retry limit explicitly, because it's set to 0 when + // --gtest_filter is passed. + '--test-launcher-retry-limit=1', + ], + + expectedLines: [ + "##teamcity[testSuiteStarted name='brave_unit_tests']", + "##teamcity[testRetrySupport enabled='true']", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Success'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Success'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testSuiteFinished name='brave_unit_tests']" + ] + } + + const ignorePreliminaryFailuresTestCase = { + args: [ + ...generalTestCase.args, + '--test-launcher-teamcity-reporter-ignore-preliminary-failures' + ], + + expectedLines: [ + "##teamcity[testSuiteStarted name='brave_unit_tests']", + "##teamcity[testRetrySupport enabled='true']", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Success'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Success'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure'", + "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped'", + "##teamcity[testSuiteFinished name='brave_unit_tests']" + ] + } const runOptions = config.defaultOptions runOptions.stdio = 'pipe' runOptions.continueOnFail = true - const prog = util.run(path.join(config.outputDir, 'brave_unit_tests'), args, runOptions) - const expectedOutput = [ - "##teamcity[testSuiteStarted name='brave_unit_tests']", - "##teamcity[testRetrySupport enabled='false']", - "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Success' captureStandardOutput='true']", - "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Success' duration='", - "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Failure' captureStandardOutput='true']", - "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.Failure']", - "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Failure' duration='", - "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure' captureStandardOutput='true']", - "##teamcity[testFailed name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure']", - "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.CheckFailure' duration='", - "##teamcity[testStarted name='DISABLED_TeamcityReporterIntegrationTest.Skipped' captureStandardOutput='true']", - "##teamcity[testIgnored name='DISABLED_TeamcityReporterIntegrationTest.Skipped']", - "##teamcity[testFinished name='DISABLED_TeamcityReporterIntegrationTest.Skipped' duration='", - "##teamcity[testSuiteFinished name='brave_unit_tests']", - ] - - const stdoutLines = prog.stdout.toString().split('\n'); - for (const line of stdoutLines) { - if (line.startsWith(expectedOutput[0])) { - expectedOutput.splice(0, 1) - if (expectedOutput.length == 0) { - break - } - } + for (const testCase of [generalTestCase, ignorePreliminaryFailuresTestCase]) { + const prog = util.run( + path.join(config.outputDir, 'brave_unit_tests'), + testCase.args, + runOptions + ) + const outputLines = prog.stdout.toString().split('\n') + checkTeamcityReporterOutput(outputLines, testCase.expectedLines) } +} + +const checkTeamcityReporterOutput = (outputLines, expectedTeamcityLines) => { + const outputTeamcityLines = outputLines.filter((line) => + line.startsWith('##teamcity') + ) + + const isMatched = + outputTeamcityLines.length === expectedTeamcityLines.length && + expectedTeamcityLines.every((expectedLine, index) => { + const outputLine = outputTeamcityLines[index] + return outputLine.startsWith(expectedLine) + }) - const isMatched = expectedOutput.length == 0; if (!isMatched) { + const notMatchedOutputLines = outputTeamcityLines.filter( + (outputLine, index) => { + const expectedLine = expectedTeamcityLines[index] + return !outputLine.startsWith(expectedLine) + } + ) + + const notMatchedExpectedLines = expectedTeamcityLines.filter( + (expectedLine, index) => { + const outputLine = outputTeamcityLines[index] + return !outputLine?.startsWith(expectedLine) + } + ) + + Log.error( + 'TeamcityReporter output test failed, output ##teamcity lines do not match expected lines (note ##teamcity was replaced with %%teamcity):' + ) console.error( - prog.stdout.toString().replace(/##teamcity/gm, '%%teamcity') + - '\nChromiumTestLauncherTeamcityReporterIntegration test failed, the line was not found in the output (note: ##teamcity was replaced with %%teamcity):\n' + - expectedOutput[0].replace(/##teamcity/gm, '%%teamcity') + [ + '\nNot matched output lines:', + ...notMatchedOutputLines, + '\nNot matched expected lines:', + ...notMatchedExpectedLines, + '\nTest output lines:', + ...outputTeamcityLines, + '\nExpected lines:', + ...expectedTeamcityLines, + '\nFull test output:', + ...outputLines + ] + .join('\n') + .replace(/##teamcity/gm, '%%teamcity') ) process.exit(1) } else { - console.log('ChromiumTestLauncherTeamcityReporterIntegration test passed') + console.log('TeamcityReporter output test passed') } } diff --git a/chromium_src/base/test/launcher/test_launcher.cc b/chromium_src/base/test/launcher/test_launcher.cc index b320eaae87a..41bb236aa14 100644 --- a/chromium_src/base/test/launcher/test_launcher.cc +++ b/chromium_src/base/test/launcher/test_launcher.cc @@ -54,7 +54,7 @@ void TestLauncher::CreateAndStartThreadPool(size_t num_parallel_jobs) { // `retry_limit_` can be overridden by command line. Read its value when all // command line flags are parsed. if (teamcity_reporter_) { - teamcity_reporter_->EnableRetrySupport(retry_limit_ != 0); + teamcity_reporter_->SetRetryLimit(retry_limit_); } TestLauncher_ChromiumImpl::CreateAndStartThreadPool(num_parallel_jobs);