Discharge7443/brave-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Output android test results in the same format as desktop when using --output_xml (#29870)

Browse Source
This commit is contained in:
Brian Johnson
2025-07-15 05:23:29 +02:00
committed by GitHub
parent 70ea86dfdf
commit e77cfdaab3
4 changed files with 121 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/.ci_features
+1
View File
@@ -3,3 +3,4 @@
gn_check_supports_checkdeps_only
test_report_lists
android_output_xml
build/commands/lib/test.js
+75 -63
View File
@@ -12,7 +12,14 @@ const util = require('../lib/util')
const assert = require('assert')
const getTestBinary = (suite) => {
return process.platform === 'win32' ? `${suite}.exe` : suite
let testBinary = suite
if (testBinary === 'brave_java_unit_tests') {
testBinary = path.join('bin', 'run_brave_java_unit_tests')
} else if (testBinary === 'brave_junit_tests') {
testBinary = path.join('bin', 'run_brave_junit_tests')
}
testBinary = path.join(config.outputDir, testBinary)
return process.platform === 'win32' ? `${testBinary}.exe` : testBinary
}
const getChromiumUnitTestsSuites = () => {
@@ -25,11 +32,9 @@ const getChromiumUnitTestsSuites = () => {
}
const getBraveUnitTestsSuites = (config) => {
let tests = []
let tests = ['brave_unit_tests', 'brave_components_unittests']
if (config.targetOS !== 'android') {
// TODO(bridiver) https://github.com/brave/brave-browser/issues/47310
tests.push('brave_components_unittests')
tests.push('brave_installer_unittests')
}
@@ -37,15 +42,13 @@ const getBraveUnitTestsSuites = (config) => {
}
const getTestsToRun = (config, suite) => {
let testsToRun = [suite]
if (suite === 'brave_unit_tests') {
testsToRun = testsToRun.concat(getBraveUnitTestsSuites(config))
} else if (suite === 'brave_java_unit_tests') {
testsToRun = ['bin/run_brave_java_unit_tests']
} else if (suite === 'brave_junit_tests') {
testsToRun = ['bin/run_brave_junit_tests']
let testsToRun = []
if (suite === 'brave_all_unit_tests') {
testsToRun = [...getBraveUnitTestsSuites(config)]
} else if (suite === 'chromium_unit_tests') {
testsToRun = getChromiumUnitTestsSuites()
} else {
testsToRun = [suite]
}
return testsToRun
}
@@ -112,9 +115,7 @@ const buildTests = async (
'brave_junit_tests',
'brave_network_audit_tests',
]
if (suite === 'brave_unit_tests') {
config.buildTargets = ['all_unit_tests']
} else if (testSuites.includes(suite)) {
if (testSuites.includes(suite)) {
config.buildTargets = ['brave/test:' + suite]
} else if (suite === 'chromium_unit_tests') {
config.buildTargets = getChromiumUnitTestsSuites()
@@ -216,63 +217,33 @@ const runTests = (passthroughArgs, suite, buildConfig, options) => {
'browser_tests',
...getChromiumUnitTestsSuites(),
]
// Run the tests
getTestsToRun(config, suite).every((testSuite) => {
let runArgs = braveArgs.slice()
let runOptions = config.defaultOptions
// Filter out upstream tests that are known to fail for Brave
if (upstreamTestSuites.includes(testSuite)) {
const previousFilters = braveArgs.findIndex((arg) => {
return arg.startsWith('--test-launcher-filter-file=')
})
if (previousFilters !== -1) {
braveArgs.splice(previousFilters, 1)
}
const filterFilePaths = getApplicableFilters(testSuite)
if (filterFilePaths.length > 0) {
braveArgs.push(
runArgs.push(
`--test-launcher-filter-file=${filterFilePaths.join(';')}`,
)
}
if (config.isTeamcity) {
const ignorePreliminaryFailures =
'--test-launcher-teamcity-reporter-ignore-preliminary-failures'
if (!braveArgs.includes(ignorePreliminaryFailures)) {
braveArgs.push(ignorePreliminaryFailures)
if (!runArgs.includes(ignorePreliminaryFailures)) {
runArgs.push(ignorePreliminaryFailures)
}
}
}
if (options.output_xml) {
const previousOutput = braveArgs.findIndex((arg) => {
return arg.startsWith('--gtest_output=xml:')
})
if (previousOutput !== -1) {
braveArgs.splice(previousOutput, 1)
}
braveArgs.push(`--gtest_output=xml:${testSuite}.xml`)
}
if (config.targetOS === 'android' && !isJunitTestSuite) {
assert(
config.targetArch === 'x86'
|| config.targetArch === 'x64'
|| options.manual_android_test_device,
'Only x86 and x64 builds can be run automatically. For other builds please run test device manually and specify manual_android_test_device flag.',
)
}
if (
config.targetOS === 'android'
&& !isJunitTestSuite
&& !options.manual_android_test_device
) {
// Specify emulator to run tests on
braveArgs.push(
`--avd-config=tools/android/avd/proto/${options.android_test_emulator_name}.textpb`,
)
}
let runOptions = config.defaultOptions
if (config.isTeamcity) {
// Stdout and stderr must be separate for a test launcher.
runOptions.stdio = 'inherit'
}
if (options.output_xml) {
let convertJSONToXML = false
let outputFilename = path.join(config.srcDir, testSuite)
if (config.isCI) {
// When test results are saved to a file, callers (such as CI) generate
// and analyze test reports as a next step. These callers are typically
// not interested in the exit code of running the tests, because they
@@ -283,17 +254,58 @@ const runTests = (passthroughArgs, suite, buildConfig, options) => {
// failures (by looking at the output file) from compilation errors.
runOptions.continueOnFail = true
}
let prog = util.run(
path.join(config.outputDir, getTestBinary(testSuite)),
braveArgs,
runOptions,
)
if (options.output_xml) {
// Add filename of xml output of each test suite into the results file
if (config.targetOS === 'android') {
// android only supports json output so use that here and convert
// to xml afterwards
runArgs.push(`--json-results-file=${outputFilename}.json`)
convertJSONToXML = true
} else {
runArgs.push(`--gtest_output=xml:${outputFilename}.xml`)
}
fs.appendFileSync(allResultsFilePath, `${testSuite}.xml\n`)
}
// Don't run other tests if one has failed already.
return prog.status === 0
if (config.targetOS === 'android' && !isJunitTestSuite) {
assert(
config.targetArch === 'x86'
|| config.targetArch === 'x64'
|| options.manual_android_test_device,
'Only x86 and x64 builds can be run automatically. For other builds please run test device manually and specify manual_android_test_device flag.',
)
if (!options.manual_android_test_device) {
runArgs.push(
`--avd-config=tools/android/avd/proto/${options.android_test_emulator_name}.textpb`,
)
}
}
if (config.isTeamcity) {
// Stdout and stderr must be separate for a test launcher.
runOptions.stdio = 'inherit'
}
let prog = util.run(getTestBinary(testSuite), runArgs, runOptions)
// convert json results to xml
if (convertJSONToXML) {
prog = util.run('vpython3', [path.join('script', 'json2xunit.py')], {
...config.defaultOptions,
cwd: config.braveCoreDir,
stdio: [
fs.openSync(`${outputFilename}.json`, 'r'),
fs.openSync(`${outputFilename}.xml`, 'w'),
'inherit',
],
})
}
// If we output results into an xml file (CI), then we want to run all
// suites to get all potential failures. Otherwise, for example, if
// running locally, it makes sense to stop once one suite has failures.
return options.output_xml || prog.status === 0
})
}
}
script/json2xunit.py
+44 -20
View File
@@ -1,5 +1,9 @@
#!/usr/bin/env python3
# pylint:disable=line-too-long,consider-using-dict-items
#!/usr/bin/env vpython3
# Copyright (c) 2021 The Brave Authors. All rights reserved.
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at https://mozilla.org/MPL/2.0/.
"""
On Android the `--output' switch to `npm run test', which produces xunit
@@ -29,41 +33,55 @@ def pick_iteration(test_case, iterations):
def score(iteration):
score = 1000000 if iteration['status'] == 'SUCCESS' else 0
score += len(iteration['output_snippet'])
started_this_test = re.compile(
r'\[ RUN \] [\w/#\.]*' + re.escape(test_case) + '\\n')
started_this_test = re.compile(r'\[ RUN \] [\w/#\.]*' +
re.escape(test_case) + '\\n')
score += 7500 if started_this_test.match(
iteration['output_snippet']) else 0
for string in ['Stack Trace:', 'Expected:', 'Actual:']:
score += 2500 if string in iteration['output_snippet'] else 0
return score
return sorted(iterations, key=score, reverse=True)[0]
def transform(input_json):
"""Read json input and return a dictionary with information necessary to produce the xunit output"""
"""Read json input and return a dictionary with information necessary to
produce the xunit output"""
output = collections.defaultdict(lambda: {
'xml': '',
'test_count': 0,
'failure_count': 0
})
if not input_json['per_iteration_data']:
return output
output = collections.defaultdict(
lambda: {'xml': '', 'test_count': 0, 'failure_count': 0})
test_results = input_json['per_iteration_data'][0]
for test_fullname, iterations in test_results.items():
delim = '#' if '#' in test_fullname else '/' if '/' in test_fullname else '.'
delim = ('#' if '#' in test_fullname else
'/' if '/' in test_fullname else '.')
test_suite, test_case = test_fullname.rsplit(delim, maxsplit=1)
iteration = pick_iteration(test_case, iterations)
output[test_suite]['test_count'] += 1
output[test_suite][
'xml'] += f'<testcase name="{test_case}" time="{int(iteration["elapsed_time_ms"])/100.0}">'
output[test_suite]['xml'] += (
f'<testcase name="{test_case}" '
f'time="{int(iteration["elapsed_time_ms"])/100.0}">')
if iteration['status'] == 'SUCCESS':
if iteration['output_snippet']:
sanitized_output = ''.join(
filter(lambda x: x in printable, iteration['output_snippet']))
output[test_suite]['xml'] += f"<system-out><![CDATA[{sanitized_output}]]></system-out>"
filter(lambda x: x in printable,
iteration['output_snippet']))
output[test_suite]['xml'] += (
f"<system-out><![CDATA["
f"{sanitized_output}]]></system-out>")
else:
output[test_suite]['failure_count'] += 1
sanitized_output = ''.join(
filter(lambda x: x in printable, iteration['output_snippet']))
output[test_suite][
'xml'] += f'<failure message="failed"><![CDATA[{sanitized_output}]]></failure>'
output[test_suite]['xml'] += (
f'<failure message="failed"><![CDATA['
f'{sanitized_output}]]></failure>')
output[test_suite]['xml'] += "</testcase>"
return output
@@ -73,14 +91,20 @@ def main():
output = transform(json.load(sys.stdin))
test_count = reduce(add, (ts['test_count'] for ts in output.values()), 0)
failure_count = reduce(add, (ts['failure_count']
for ts in output.values()), 0)
failure_count = reduce(add,
(ts['failure_count'] for ts in output.values()), 0)
print(f"""<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="tests" """
f"""tests="{test_count}" errors="0" failures="{failure_count}" skip="0">""", end='')
print(
f'<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="tests" '
f'tests="{test_count}" errors="0" failures="{failure_count}" skip="0">',
end='')
for test_suite in output.keys():
print(f'<testsuite name="{test_suite}" tests="{output[test_suite]["test_count"]}" errors="0" failures='
f'"{output[test_suite]["failure_count"]}" skip="0">{output[test_suite]["xml"]}</testsuite>', end='')
print(
f'<testsuite name="{test_suite}" '
f'tests="{output[test_suite]["test_count"]}" errors="0" failures='
f'"{output[test_suite]["failure_count"]}" '
f'skip="0">{output[test_suite]["xml"]}</testsuite>',
end='')
print('</testsuites>', end='')
test/BUILD.gn
+1 -1
View File
@@ -87,7 +87,7 @@ static_library("brave_test_support_unit") {
}
}
group("all_unit_tests") {
group("brave_all_unit_tests") {
testonly = true
data_deps = [