Smoke perf tests (#30310)

The PR supports command `npm run perf_tests -- smoke Static` to verify that perf tests code work as expected. This will be used to verify some metrics in future.
2025-07-31 16:15:53 +04:00
parent 310be28ded
commit e35cbdae11
8 changed files with 125 additions and 27 deletions
@@ -6,23 +6,54 @@
 const util = require('../lib/util')
 const config = require('../lib/config')
 const path = require('path')
+const fs = require('fs')

-const runPerfTests = (passthroughArgs, perfConfig, targets) => {
+const runPerfTests = (passthroughArgs, perfConfig, targetBuildConfig) => {
  args = [
    path.join(config.braveCoreDir, 'tools', 'perf', 'run_perftests.py'),
    perfConfig,
  ]

-  if (targets !== undefined) {
+  if (['Static', 'Release', 'Component'].includes(targetBuildConfig)) {
+    config.buildConfig = targetBuildConfig
+    config.update({})
+
+    binaryPath = path.join(config.outputDir, 'brave')
    if (process.platform === 'win32') {
-      targets = '"' + targets + '"'
+      binaryPath += '.exe'
+    } else if (process.platform === 'darwin') {
+      binaryPath = fs
+        .readFileSync(binaryPath + '_helper')
+        .toString()
+        .trim()
+
+      // Convert "\ " to " ":
+      binaryPath = binaryPath.replace(/\\ /g, ' ')
    }
-    args.push(targets)
+    const braveCoreCommit = util.runGit(
+      config.braveCoreDir,
+      ['rev-parse', 'HEAD'],
+      true,
+    )
+    targetBuildConfig = `${braveCoreCommit}:${binaryPath}`
+  }
+
+  if (targetBuildConfig !== undefined) {
+    args.push(targetBuildConfig)
  }

  args.push(...passthroughArgs)
  console.log(args)
-  util.run('vpython3', args, config.defaultOptions)
+
+  let cmdOptions = {
+    ...config.defaultOptions,
+    shell: false,
+  }
+  if (process.platform === 'win32') {
+    util.run('cmd.exe', ['/c', 'vpython3.bat', ...args], cmdOptions)
+  } else {
+    util.run('vpython3', args, cmdOptions)
+  }
 }

 module.exports = { runPerfTests }
@@ -16,7 +16,7 @@ from components.common_options import CommonOptions
 from components.perf_test_utils import (DownloadArchiveAndUnpack, DownloadFile,
                                        GetProcessOutput, ToBravePlatformName,
                                        ToChromiumPlatformName)
-from components.version import BraveVersion
+from components.version import CHROME_VERSION_RE_PATTERN, BraveVersion


 def _GetBraveDownloadUrl(tag: str, filename: str) -> str:
@@ -56,7 +56,7 @@ def _DownloadWinInstallerAndExtract(out_dir: str, url: str,
  logging.info('Copy files to %s', out_dir)
  copy_tree(expected_install_path, out_dir)
  for file in os.listdir(expected_install_path):
-    if re.match(r'\d+\.\d+\.\d+.\d+', file):
+    if re.match(CHROME_VERSION_RE_PATTERN, file):
      assert (full_version is None)
      full_version = file
  assert (full_version is not None)
@@ -8,6 +8,7 @@
 from enum import Enum

 import argparse
+import logging
 import os
 import sys
 import tempfile
@@ -63,7 +64,7 @@ class CommonOptions:
        type=str,
        help='The path/URL to a config. See configs/**/.json for examples.'
        'Also could be set to "auto" to select the config by '
-        'machine-id + chromium')
+        'machine-id + chromium or "smoke" to run the smoke tests.')
    parser.add_argument(
        'targets',
        type=str,
@@ -146,6 +147,24 @@ class CommonOptions:
  @classmethod
  def from_args(cls, args) -> 'CommonOptions':
    options = CommonOptions()
+    options.config = args.config
+    if options.config == 'auto':  # Select the config by machine_id and chromium
+      if options.machine_id is None:
+        raise RuntimeError('Set --machine-id to use config=auto')
+
+      prefix = 'chromium' if options.chromium else 'brave'
+      config = (f'{prefix}-{options.target_os}-' +
+                f'{options.target_arch}-{options.machine_id}.json5')
+      logging.info('Using %s as config=auto', config)
+      options.config = os.path.join(path_util.GetBravePerfConfigDir(), 'ci',
+                                    config)
+    elif options.config == 'smoke':
+      args.no_report = True
+      options.config = os.path.join(path_util.GetBravePerfConfigDir(),
+                                    'smoke.json5')
+      args.working_directory = os.path.join(path_util.GetSrcDir(),
+                                            'perf-test-smoke')
+
    if args.working_directory is None:
      if options.ci_mode:
        raise RuntimeError('Set --working-directory for --ci-mode')
@@ -14,6 +14,7 @@ from enum import Enum

 from components.field_trials import FieldTrialsMode, ParseFieldTrialsMode
 from components.browser_type import BrowserType, ParseBrowserType
+from components.perf_test_utils import GetProcessOutput
 from components.version import BraveVersion


@@ -68,6 +69,19 @@ class RunnerConfig:
        raise RuntimeError(f'Unexpected {key} in configuration')
      setattr(self, key_, json[key])

+
+def ParseVersionFromBinary(binary: str) -> BraveVersion:
+  _, output = GetProcessOutput([binary, '--version'])
+  logging.info('Get binary version: %s for %s', output.strip(), binary)
+
+  version = re.search(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', output.strip())
+  if version is None:
+    raise RuntimeError(f'Can parse version from binary {output}')
+
+  version = version.group(0).split('.')[1:]
+  return BraveVersion('v' + '.'.join(version))
+
+
 def ParseTarget(target: str) -> Tuple[Optional[BraveVersion], str]:
  """
  Parse the version and location from the passed string `target`.
@@ -76,8 +90,12 @@ def ParseTarget(target: str) -> Tuple[Optional[BraveVersion], str]:
  1. Brave tag (i.e. v1.62.1);
  2. Git hash;
  3. empty (for comparing builds when you don't need it).
+  4. binary path (version is parsed from the binary output)
  """

+  if os.path.exists(target):
+    return ParseVersionFromBinary(target), target
+
  m = re.match(r'^(v\d+\.\d+\.\d+|\w+)(?::(.+)|$)', target)
  if not m:
    return None, target
@@ -98,6 +116,8 @@ class BenchmarkConfig:
  pageset_repeat: int = 1
  stories: List[str]
  stories_exclude: List[str]
+  extra_benchmark_args: List[str] = []
+  extra_browser_args: List[str] = []

  def __init__(self, json: Optional[dict] = None):
    if not json:
@@ -108,6 +128,8 @@ class BenchmarkConfig:
      self.pageset_repeat = pageset_repeat
    self.stories = json.get('stories') or []
    self.stories_exclude = json.get('stories-exclude') or []
+    self.extra_benchmark_args = json.get('extra-benchmark-args') or []
+    self.extra_browser_args = json.get('extra-browser-args') or []


 class PerfConfig:
@@ -179,6 +179,10 @@ class RunableConfiguration:
    args.extend(binary.get_run_benchmark_args())
    browser_args.extend(self.binary.get_browser_args())

+    # process the benchmark-specific args:
+    args.extend(benchmark_config.extra_benchmark_args)
+    browser_args.extend(benchmark_config.extra_browser_args)
+
    # process the extra args from json config:
    args.extend(self.config.extra_benchmark_args)
    browser_args.extend(self.config.extra_browser_args)
@@ -370,7 +374,7 @@ def SpawnConfigurationsFromTargetList(target_list: List[str],
    if not config.location:
      config.location = location
    if not config.version:
-      raise RuntimeError(f'Can get the version from target {target_string}')
+      raise RuntimeError(f'Can get the version from target "{target_string}"')
    if not config.label:
      config.label = config.version.to_string()
    configurations.append(config)
@@ -9,13 +9,15 @@ import json
 from typing import List
 import components.git_tools as git_tools

+CHROME_VERSION_RE_PATTERN = r'\d{1,4}\.\d{1,4}\.\d{1,4}\.\d{1,4}'
+BRAVE_VERSION_RE_PATTERN = r'v\d{1,4}\.\d{1,4}\.\d{1,4}'

 class ChromiumVersion:
  _version: List[int]

  def __init__(self, v: str) -> None:
    super().__init__()
-    assert re.match(r'\d+\.\d+\.\d+\.\d+', v)
+    assert re.match(CHROME_VERSION_RE_PATTERN, v)
    self._version = list(map(int, v.split('.')))

  def to_string(self) -> str:
@@ -34,7 +36,7 @@ class BraveVersion:
  _chromium_version: ChromiumVersion

  def __init__(self, version_str: str) -> None:
-    m = re.match(r'v\d+\.\d+\.\d+', version_str)
+    m = re.match(BRAVE_VERSION_RE_PATTERN, version_str)
    if m is not None:  # Brave tag (v1.62.35)
      revision = f'refs/tags/{version_str}'
      self._is_tag = True
@@ -0,0 +1,31 @@
+{
+  "configurations": [{
+    "browser-type": "brave",
+    "profile": "brave-typical-mac",
+    "extra-browser-args": [
+      "--disable-component-update",
+      "--disable-backgrounding-occluded-windows",
+    ],
+    "field-trials": "no-trials",
+    "save-artifacts": true,
+  }],
+  "benchmarks": [
+    {
+      "name": "system_health.common_desktop",
+      "pageset-repeat": 1,
+      "stories": [
+        "load:site:example:2023",
+      ],
+      "extra-benchmark-args": [
+        "--allow-software-compositing",
+      ],
+    },
+    {
+      "name": "system_health.memory_desktop",
+      "pageset-repeat": 1,
+      "stories": [
+        "load:site:example:2023",
+      ],
+    },
+  ]
+}
@@ -41,7 +41,8 @@ with path_util.SysPath(path_util.GetPyJson5Dir()):
  # pylint: enable=import-error # pytype: enable=import-error


-def load_config(config: str, options: CommonOptions) -> dict:
+def load_config(options: CommonOptions) -> dict:
+  config = options.config
  if config.startswith('https://'):  # URL to download the config
    _, config_path = tempfile.mkstemp(dir=options.working_directory,
                                      prefix='config-')
@@ -49,18 +50,6 @@ def load_config(config: str, options: CommonOptions) -> dict:

  elif os.path.isfile(config):  # Full config path
    config_path = config
-  elif config == 'auto':  # Select the config by machine_id and chromium
-    if options.machine_id is None:
-      raise RuntimeError('Set --machine-id to use config=auto')
-
-    prefix = 'chromium' if options.chromium else 'brave'
-    config = (f'{prefix}-{options.target_os}-' +
-              f'{options.target_arch}-{options.machine_id}.json5')
-    logging.info('Using %s as config=auto', config)
-    config_path = os.path.join(path_util.GetBravePerfConfigDir(), 'ci', config)
-
-    if not os.path.isfile(config_path):
-      raise RuntimeError(f'No config file {config_path}')
  else:  # config is a relative path
    config_path = os.path.join(path_util.GetBravePerfConfigDir(), config)
    if not os.path.isfile(config_path):
@@ -100,7 +89,7 @@ npm run perf_tests -- smoke-brave.json5 v1.58.45

  os.makedirs(options.working_directory, exist_ok=True)

-  json_config = load_config(args.config, options)
+  json_config = load_config(options)
  config = perf_config.PerfConfig(json_config)

  if options.is_android:
@@ -125,10 +114,10 @@ npm run perf_tests -- smoke-brave.json5 v1.58.45
    if options.chromium:
      return 0  # A build with !options.chromium will update the both profiles
    options.chromium = True
-    chromium_config = perf_config.PerfConfig(load_config(args.config, options))
+    chromium_config = perf_config.PerfConfig(load_config(options))
    chromium_config.runners[0].label = 'chromium-rebase'
    options.chromium = False
-    brave_config = perf_config.PerfConfig(load_config(args.config, options))
+    brave_config = perf_config.PerfConfig(load_config(options))
    brave_config.runners[0].label = 'brave-rebase'
    return 0 if profile_tools.RunUpdateProfile(brave_config, chromium_config,
                                               options) else 1