From e35cbdae11d9a250254783f0f3725ba387fde665 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Thu, 31 Jul 2025 16:15:53 +0400 Subject: [PATCH] Smoke perf tests (#30310) The PR supports command `npm run perf_tests -- smoke Static` to verify that perf tests code work as expected. This will be used to verify some metrics in future. --- build/commands/lib/perfTests.js | 41 ++++++++++++++++++++--- tools/perf/components/browser_type.py | 4 +-- tools/perf/components/common_options.py | 21 +++++++++++- tools/perf/components/perf_config.py | 22 ++++++++++++ tools/perf/components/perf_test_runner.py | 6 +++- tools/perf/components/version.py | 6 ++-- tools/perf/configs/smoke.json5 | 31 +++++++++++++++++ tools/perf/run_perftests.py | 21 +++--------- 8 files changed, 125 insertions(+), 27 deletions(-) create mode 100644 tools/perf/configs/smoke.json5 diff --git a/build/commands/lib/perfTests.js b/build/commands/lib/perfTests.js index 7289f05ab65..6a084ae2b01 100644 --- a/build/commands/lib/perfTests.js +++ b/build/commands/lib/perfTests.js @@ -6,23 +6,54 @@ const util = require('../lib/util') const config = require('../lib/config') const path = require('path') +const fs = require('fs') -const runPerfTests = (passthroughArgs, perfConfig, targets) => { +const runPerfTests = (passthroughArgs, perfConfig, targetBuildConfig) => { args = [ path.join(config.braveCoreDir, 'tools', 'perf', 'run_perftests.py'), perfConfig, ] - if (targets !== undefined) { + if (['Static', 'Release', 'Component'].includes(targetBuildConfig)) { + config.buildConfig = targetBuildConfig + config.update({}) + + binaryPath = path.join(config.outputDir, 'brave') if (process.platform === 'win32') { - targets = '"' + targets + '"' + binaryPath += '.exe' + } else if (process.platform === 'darwin') { + binaryPath = fs + .readFileSync(binaryPath + '_helper') + .toString() + .trim() + + // Convert "\ " to " ": + binaryPath = binaryPath.replace(/\\ /g, ' ') } - args.push(targets) + const braveCoreCommit = util.runGit( + config.braveCoreDir, + ['rev-parse', 'HEAD'], + true, + ) + targetBuildConfig = `${braveCoreCommit}:${binaryPath}` + } + + if (targetBuildConfig !== undefined) { + args.push(targetBuildConfig) } args.push(...passthroughArgs) console.log(args) - util.run('vpython3', args, config.defaultOptions) + + let cmdOptions = { + ...config.defaultOptions, + shell: false, + } + if (process.platform === 'win32') { + util.run('cmd.exe', ['/c', 'vpython3.bat', ...args], cmdOptions) + } else { + util.run('vpython3', args, cmdOptions) + } } module.exports = { runPerfTests } diff --git a/tools/perf/components/browser_type.py b/tools/perf/components/browser_type.py index be374c70b03..3d936c1a7af 100644 --- a/tools/perf/components/browser_type.py +++ b/tools/perf/components/browser_type.py @@ -16,7 +16,7 @@ from components.common_options import CommonOptions from components.perf_test_utils import (DownloadArchiveAndUnpack, DownloadFile, GetProcessOutput, ToBravePlatformName, ToChromiumPlatformName) -from components.version import BraveVersion +from components.version import CHROME_VERSION_RE_PATTERN, BraveVersion def _GetBraveDownloadUrl(tag: str, filename: str) -> str: @@ -56,7 +56,7 @@ def _DownloadWinInstallerAndExtract(out_dir: str, url: str, logging.info('Copy files to %s', out_dir) copy_tree(expected_install_path, out_dir) for file in os.listdir(expected_install_path): - if re.match(r'\d+\.\d+\.\d+.\d+', file): + if re.match(CHROME_VERSION_RE_PATTERN, file): assert (full_version is None) full_version = file assert (full_version is not None) diff --git a/tools/perf/components/common_options.py b/tools/perf/components/common_options.py index 3576c0a1643..e29764dcfa7 100644 --- a/tools/perf/components/common_options.py +++ b/tools/perf/components/common_options.py @@ -8,6 +8,7 @@ from enum import Enum import argparse +import logging import os import sys import tempfile @@ -63,7 +64,7 @@ class CommonOptions: type=str, help='The path/URL to a config. See configs/**/.json for examples.' 'Also could be set to "auto" to select the config by ' - 'machine-id + chromium') + 'machine-id + chromium or "smoke" to run the smoke tests.') parser.add_argument( 'targets', type=str, @@ -146,6 +147,24 @@ class CommonOptions: @classmethod def from_args(cls, args) -> 'CommonOptions': options = CommonOptions() + options.config = args.config + if options.config == 'auto': # Select the config by machine_id and chromium + if options.machine_id is None: + raise RuntimeError('Set --machine-id to use config=auto') + + prefix = 'chromium' if options.chromium else 'brave' + config = (f'{prefix}-{options.target_os}-' + + f'{options.target_arch}-{options.machine_id}.json5') + logging.info('Using %s as config=auto', config) + options.config = os.path.join(path_util.GetBravePerfConfigDir(), 'ci', + config) + elif options.config == 'smoke': + args.no_report = True + options.config = os.path.join(path_util.GetBravePerfConfigDir(), + 'smoke.json5') + args.working_directory = os.path.join(path_util.GetSrcDir(), + 'perf-test-smoke') + if args.working_directory is None: if options.ci_mode: raise RuntimeError('Set --working-directory for --ci-mode') diff --git a/tools/perf/components/perf_config.py b/tools/perf/components/perf_config.py index a04908b7647..52a7aaeab81 100644 --- a/tools/perf/components/perf_config.py +++ b/tools/perf/components/perf_config.py @@ -14,6 +14,7 @@ from enum import Enum from components.field_trials import FieldTrialsMode, ParseFieldTrialsMode from components.browser_type import BrowserType, ParseBrowserType +from components.perf_test_utils import GetProcessOutput from components.version import BraveVersion @@ -68,6 +69,19 @@ class RunnerConfig: raise RuntimeError(f'Unexpected {key} in configuration') setattr(self, key_, json[key]) + +def ParseVersionFromBinary(binary: str) -> BraveVersion: + _, output = GetProcessOutput([binary, '--version']) + logging.info('Get binary version: %s for %s', output.strip(), binary) + + version = re.search(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', output.strip()) + if version is None: + raise RuntimeError(f'Can parse version from binary {output}') + + version = version.group(0).split('.')[1:] + return BraveVersion('v' + '.'.join(version)) + + def ParseTarget(target: str) -> Tuple[Optional[BraveVersion], str]: """ Parse the version and location from the passed string `target`. @@ -76,8 +90,12 @@ def ParseTarget(target: str) -> Tuple[Optional[BraveVersion], str]: 1. Brave tag (i.e. v1.62.1); 2. Git hash; 3. empty (for comparing builds when you don't need it). + 4. binary path (version is parsed from the binary output) """ + if os.path.exists(target): + return ParseVersionFromBinary(target), target + m = re.match(r'^(v\d+\.\d+\.\d+|\w+)(?::(.+)|$)', target) if not m: return None, target @@ -98,6 +116,8 @@ class BenchmarkConfig: pageset_repeat: int = 1 stories: List[str] stories_exclude: List[str] + extra_benchmark_args: List[str] = [] + extra_browser_args: List[str] = [] def __init__(self, json: Optional[dict] = None): if not json: @@ -108,6 +128,8 @@ class BenchmarkConfig: self.pageset_repeat = pageset_repeat self.stories = json.get('stories') or [] self.stories_exclude = json.get('stories-exclude') or [] + self.extra_benchmark_args = json.get('extra-benchmark-args') or [] + self.extra_browser_args = json.get('extra-browser-args') or [] class PerfConfig: diff --git a/tools/perf/components/perf_test_runner.py b/tools/perf/components/perf_test_runner.py index fa83f0112db..9ee5c411aa0 100644 --- a/tools/perf/components/perf_test_runner.py +++ b/tools/perf/components/perf_test_runner.py @@ -179,6 +179,10 @@ class RunableConfiguration: args.extend(binary.get_run_benchmark_args()) browser_args.extend(self.binary.get_browser_args()) + # process the benchmark-specific args: + args.extend(benchmark_config.extra_benchmark_args) + browser_args.extend(benchmark_config.extra_browser_args) + # process the extra args from json config: args.extend(self.config.extra_benchmark_args) browser_args.extend(self.config.extra_browser_args) @@ -370,7 +374,7 @@ def SpawnConfigurationsFromTargetList(target_list: List[str], if not config.location: config.location = location if not config.version: - raise RuntimeError(f'Can get the version from target {target_string}') + raise RuntimeError(f'Can get the version from target "{target_string}"') if not config.label: config.label = config.version.to_string() configurations.append(config) diff --git a/tools/perf/components/version.py b/tools/perf/components/version.py index 6b90e599f8e..a36565192f7 100644 --- a/tools/perf/components/version.py +++ b/tools/perf/components/version.py @@ -9,13 +9,15 @@ import json from typing import List import components.git_tools as git_tools +CHROME_VERSION_RE_PATTERN = r'\d{1,4}\.\d{1,4}\.\d{1,4}\.\d{1,4}' +BRAVE_VERSION_RE_PATTERN = r'v\d{1,4}\.\d{1,4}\.\d{1,4}' class ChromiumVersion: _version: List[int] def __init__(self, v: str) -> None: super().__init__() - assert re.match(r'\d+\.\d+\.\d+\.\d+', v) + assert re.match(CHROME_VERSION_RE_PATTERN, v) self._version = list(map(int, v.split('.'))) def to_string(self) -> str: @@ -34,7 +36,7 @@ class BraveVersion: _chromium_version: ChromiumVersion def __init__(self, version_str: str) -> None: - m = re.match(r'v\d+\.\d+\.\d+', version_str) + m = re.match(BRAVE_VERSION_RE_PATTERN, version_str) if m is not None: # Brave tag (v1.62.35) revision = f'refs/tags/{version_str}' self._is_tag = True diff --git a/tools/perf/configs/smoke.json5 b/tools/perf/configs/smoke.json5 new file mode 100644 index 00000000000..c8dbfd0f3d1 --- /dev/null +++ b/tools/perf/configs/smoke.json5 @@ -0,0 +1,31 @@ +{ + "configurations": [{ + "browser-type": "brave", + "profile": "brave-typical-mac", + "extra-browser-args": [ + "--disable-component-update", + "--disable-backgrounding-occluded-windows", + ], + "field-trials": "no-trials", + "save-artifacts": true, + }], + "benchmarks": [ + { + "name": "system_health.common_desktop", + "pageset-repeat": 1, + "stories": [ + "load:site:example:2023", + ], + "extra-benchmark-args": [ + "--allow-software-compositing", + ], + }, + { + "name": "system_health.memory_desktop", + "pageset-repeat": 1, + "stories": [ + "load:site:example:2023", + ], + }, + ] +} diff --git a/tools/perf/run_perftests.py b/tools/perf/run_perftests.py index 5fb67ae3f1f..56476b7e92e 100755 --- a/tools/perf/run_perftests.py +++ b/tools/perf/run_perftests.py @@ -41,7 +41,8 @@ with path_util.SysPath(path_util.GetPyJson5Dir()): # pylint: enable=import-error # pytype: enable=import-error -def load_config(config: str, options: CommonOptions) -> dict: +def load_config(options: CommonOptions) -> dict: + config = options.config if config.startswith('https://'): # URL to download the config _, config_path = tempfile.mkstemp(dir=options.working_directory, prefix='config-') @@ -49,18 +50,6 @@ def load_config(config: str, options: CommonOptions) -> dict: elif os.path.isfile(config): # Full config path config_path = config - elif config == 'auto': # Select the config by machine_id and chromium - if options.machine_id is None: - raise RuntimeError('Set --machine-id to use config=auto') - - prefix = 'chromium' if options.chromium else 'brave' - config = (f'{prefix}-{options.target_os}-' + - f'{options.target_arch}-{options.machine_id}.json5') - logging.info('Using %s as config=auto', config) - config_path = os.path.join(path_util.GetBravePerfConfigDir(), 'ci', config) - - if not os.path.isfile(config_path): - raise RuntimeError(f'No config file {config_path}') else: # config is a relative path config_path = os.path.join(path_util.GetBravePerfConfigDir(), config) if not os.path.isfile(config_path): @@ -100,7 +89,7 @@ npm run perf_tests -- smoke-brave.json5 v1.58.45 os.makedirs(options.working_directory, exist_ok=True) - json_config = load_config(args.config, options) + json_config = load_config(options) config = perf_config.PerfConfig(json_config) if options.is_android: @@ -125,10 +114,10 @@ npm run perf_tests -- smoke-brave.json5 v1.58.45 if options.chromium: return 0 # A build with !options.chromium will update the both profiles options.chromium = True - chromium_config = perf_config.PerfConfig(load_config(args.config, options)) + chromium_config = perf_config.PerfConfig(load_config(options)) chromium_config.runners[0].label = 'chromium-rebase' options.chromium = False - brave_config = perf_config.PerfConfig(load_config(args.config, options)) + brave_config = perf_config.PerfConfig(load_config(options)) brave_config.runners[0].label = 'brave-rebase' return 0 if profile_tools.RunUpdateProfile(brave_config, chromium_config, options) else 1