Support native RBE in Siso. (#32719)

* Support native RBE in Siso. * Move sisoenv configuration to gclient custom vars. * Small adjustments to generation/docs.
2025-12-15 13:16:44 +07:00
parent 4b86330bc8
commit d1378a7989
8 changed files with 314 additions and 9 deletions
@@ -1,2 +1,4 @@
 # This file contains supported CI features in the current checkout to
 # conditionally enable them on CI. Use lower_snake_case naming.
+
+siso_native_rbe
@@ -181,6 +181,8 @@ const Config = function () {
  this.defaultGClientFile = path.join(this.rootDir, '.gclient')
  this.gClientFile = process.env.BRAVE_GCLIENT_FILE || this.defaultGClientFile
  this.gClientVerbose = getEnvConfig(['gclient_verbose']) || false
+  this.gClientCustomDeps = getEnvConfig(['gclient_custom_deps'], {})
+  this.gClientCustomVars = getEnvConfig(['gclient_custom_vars'], {})
  this.hostOS = getHostOS()
  this.targetArch = getEnvConfig(['target_arch']) || process.arch
  this.targetOS = getEnvConfig(['target_os'])
@@ -206,6 +208,18 @@ const Config = function () {
  this.rbeService = getEnvConfig(['rbe_service']) || ''
  this.rbeTlsClientAuthCert = getEnvConfig(['rbe_tls_client_auth_cert']) || ''
  this.rbeTlsClientAuthKey = getEnvConfig(['rbe_tls_client_auth_key']) || ''
+  if (this.rbeService) {
+    this.reapiAddress = this.rbeService
+    this.reapiBackendConfigPath = path.join(
+      this.srcDir,
+      'build',
+      'config',
+      'siso',
+      'backend_config',
+      'google.star',
+    )
+    this.reapiInstance = 'default'
+  }
  this.realRewrapperDir =
    process.env.RBE_DIR || path.join(this.srcDir, 'buildtools', 'reclient')
  this.ignore_compile_failure = false
@@ -225,6 +239,7 @@ const Config = function () {
  this.extraGnGenOpts = getEnvConfig(['brave_extra_gn_gen_opts']) || ''
  this.extraNinjaOpts = []
  this.sisoJobsLimit = undefined
+  this.sisoCacheDir = getEnvConfig(['siso_cache_dir'])
  this.braveAndroidSafeBrowsingApiKey = getEnvConfig([
    'brave_safebrowsing_api_key',
  ])
@@ -240,8 +255,9 @@ const Config = function () {
  this.braveAndroidPkcs11Provider = ''
  this.braveAndroidPkcs11Alias = ''
  this.nativeRedirectCCDir = path.join(this.srcDir, 'out', 'redirect_cc')
-  this.useRemoteExec = getEnvConfig(['use_remoteexec']) || false
-  this.offline = getEnvConfig(['offline']) || false
+  this.useRemoteExec = getEnvConfig(['use_remoteexec'], false)
+  this.useReclient = getEnvConfig(['use_reclient'], this.useRemoteExec)
+  this.offline = getEnvConfig(['offline'], false)
  this.use_libfuzzer = false
  this.androidAabToApk = false
  this.useBraveHermeticToolchain = getEnvConfig(
@@ -427,7 +443,7 @@ Config.prototype.buildArgs = function () {
    enable_cdm_host_verification: this.enableCDMHostVerification(),
    skip_signing: !this.shouldSign(),
    use_remoteexec: this.useRemoteExec,
-    use_reclient: this.useRemoteExec,
+    use_reclient: this.useReclient,
    use_siso: this.useSiso,
    use_libfuzzer: this.use_libfuzzer,
    enable_update_notifications: this.isOfficialBuild(),
@@ -1180,12 +1196,17 @@ Object.defineProperty(Config.prototype, 'defaultOptions', {
        env.RBE_use_application_default_credentials =
          env.RBE_use_application_default_credentials || true
      }
+      if (this.sisoCacheDir) {
+        // This variable is handled inside the script during sync stage:
+        // brave/third_party/reclient_configs/brave_custom/brave_custom.py
+        env.SISO_CACHE_DIR = this.sisoCacheDir
+      }
    }

    // These env vars are required during `build` stage.
    if (this.useRemoteExec) {
-      // Restrict remote execution to 160 parallel jobs.
-      const kRemoteLimit = 160
+      // Restrict remote execution jobs.
+      const kRemoteLimit = getEnvConfig(['rbe_jobs_limit'], 160)

      // Prevent depot_tools from setting lower timeouts.
      const kRbeTimeout = '10m'
@@ -42,11 +42,16 @@ function buildDefaultGClientConfig(
        'src/testing/libfuzzer/fuzzers/wasm_corpus': null,
        // chromium-variations .git takes ~4 GB, not used in the build.
        'src/third_party/chromium-variations': null,
+        ...config.gClientCustomDeps,
      },
      custom_vars: {
        'checkout_clang_coverage_tools': true,
        'checkout_pgo_profiles': config.isBraveReleaseBuild(),
        'download_reclient': true,
+        'reapi_address': config.reapiAddress,
+        'reapi_backend_config_path': config.reapiBackendConfigPath,
+        'reapi_instance': config.reapiInstance,
+        ...config.gClientCustomVars,
      },
    },
  ]
@@ -60,8 +60,9 @@ __debug = False

 # Main configuration function that sets up SISO for Brave-specific build
 # requirements by adjusting step configurations and registering custom handlers.
-def __configure(ctx, step_config, handlers):
+def __configure(ctx, step_config, filegroups, handlers):
    __remove_rules(step_config)
+    __adjust_filegroups(filegroups)
    __adjust_handlers(ctx, step_config, handlers)
    __remove_labels_from_platforms(step_config)

@@ -88,6 +89,17 @@ def __remove_rules(step_config):
    ]


+def __adjust_filegroups(filegroups):
+    # Exclude vecLib.framework, a broken symlink that breaks remote execution.
+    # This symlink is not needed for the build and can be safely excluded.
+    for _, filegroup in filegroups.items():
+        includes = filegroup.get("includes", [])
+        if "*.framework" in includes:
+            excludes = filegroup.get("excludes", [])
+            excludes.append("vecLib.framework")
+            filegroup["excludes"] = excludes
+
+
 # Configures handlers to handle chromium_src overrides and disable remote
 # execution for specific rules that require local processing.
 def __adjust_handlers(ctx, step_config, handlers):
@@ -51,3 +51,5 @@ This is only an estimate. The **github** view may differ.
    setting up and optimizing your Git environment.
 *   [Ignoring Files from Format and Presubmit Checks](ignoring_format_checks.md) -
    How to exclude files from format and presubmit checks.
+*   [Siso Customization](siso_customization.md) - How we customize Siso to work
+    for us.
@@ -0,0 +1,242 @@
+# Brave's Siso Build System Customization
+
+This document explains how we customize the upstream Chromium Siso build system
+to support our build requirements, particularly the `chromium_src` override
+mechanism.
+
+## Overview
+
+Siso is Chromium's distributed build execution system that enables remote
+compilation and caching. We extend Siso to handle our build architecture where
+files in `brave/chromium_src/` can override corresponding upstream Chromium
+files without modifying the upstream source tree.
+
+## 1. Integration Point in `//build/config/siso/main.star`
+
+The entry point for our customization is:
+
+```python
+brave_siso_config.configure(ctx, step_config, filegroups, handlers)
+```
+
+This call happens **after** all upstream Chromium configurations have been
+applied:
+
+- Platform-specific configurations (Linux, macOS, Windows)
+- Mojo code generation rules
+- Rust compilation rules
+- Blink/web platform rules
+
+By hooking in at this stage, we can modify the final configuration that Siso
+will use, ensuring our customizations take precedence over upstream defaults.
+
+## 2. Configuration Customizations
+
+### 2.1 Step Configuration (`step_config`)
+
+The step configuration defines rules for build steps to run remotely by Siso.
+Rules can adjust remote execution behavior, inputs, outputs, timeouts, and more.
+We make several adjustments to make the build system work for us:
+
+#### Rule Removal
+
+- **TypeScript rules are disabled** because our `chromium_src` mechanism
+  requires special handling of TypeScript inputs/outputs that needs to be
+  implemented. See https://github.com/brave/brave-browser/issues/48135
+- **Rust rules are disabled on non-Linux hosts** for remote execution. Unlike
+  Clang (where we can use Linux toolchains for cross-compilation), Rust's Linux
+  toolchain doesn't include cross-compilation support for macOS and Windows
+  targets. Running these remotely requires Windows and macOS workers, which we
+  don't have.
+
+#### Timeout Adjustments
+
+Build nodes are typically single-core, so we give some build steps extra time to
+complete.
+
+- **Clang rules get 10-minute timeouts**
+- **Mojo and Blink Python rules get 15-minute timeouts**
+
+#### Platform Label Cleanup
+
+The current remote execution backend we use doesn't support certain Chromium
+platform labels like `action_default` and `action_large`. Removing these
+prevents remote execution failures due to unrecognized labels.
+
+### 2.2 Filegroup Customizations
+
+Siso includes all macOS and iOS frameworks by default. We exclude some
+frameworks that are not actually used by the build, but lead to remote execution
+failures:
+
+- `vecLib.framework` from macOS frameworks. This is a non-canonical symlink in
+  the macOS SDK that prevents remote system from correctly setting up remote
+  workspace. Since it's not used by the build, we can safely exclude it.
+
+### 2.3 Handler Customizations
+
+Handlers are the custom logic that processes individual build actions. We add
+two handlers:
+
+#### The `redirect_cc` Handler
+
+This handler intercepts C/C++/Objective-C compilation commands and checks if a
+corresponding override file exists in `brave/chromium_src/`.
+
+For example, when compiling `chrome/browser/ui/browser.cc`:
+
+1. Siso receives a compilation command with input
+   `../../chrome/browser/ui/browser.cc`
+2. The `redirect_cc` handler checks if
+   `../../brave/chromium_src/chrome/browser/ui/browser.cc` exists
+3. If it exists, the handler:
+   - Replaces the source file argument with the override path
+   - Adds the override file to the action's input list (so it gets uploaded for
+     remote execution)
+   - Updates the command accordingly
+
+On macOS and Windows development machines, the handler also:
+
+- Sets up a remote wrapper script that ensures Linux clang toolchain is used
+  instead of the macOS/Windows toolchain
+- Adds the Linux clang binary as an executable input
+
+This enables remote compilation on Linux workers even when building from
+non-Linux hosts.
+
+#### The `chromium_src_inputs` Handler (for Python actions)
+
+Python-based build actions can have chromium_src overrides. Overrides can exist
+for python files and also for input files. This handler ensures that the build
+system knows about all the potential file overrides and uploads them for remote
+execution:
+
+1. Scans all Python action inputs for files with supported extensions (`.mojom`,
+   `.pdl`, `.json`, `.html`, etc.)
+2. For each input file, checks if a `brave/chromium_src/` override exists
+3. Adds all found overrides to the action's input list
+4. Ensures Brave utility scripts (`brave_chromium_utils.py`,
+   `override_utils.py`) are uploaded for remote execution
+
+The handler also attaches a remote wrapper script that:
+
+- Sets `PYTHONPATH` to include Brave's script directories
+- Fixes path separators when commands originate from Windows hosts
+
+## 3. Siso Configuration Files: `.sisoenv` and `.sisorc`
+
+### `.sisoenv` - Remote Execution Service Configuration
+
+This file is generated by upstream Chromium's `configure_siso.py` script during
+`gclient runhooks`. The configuration comes from `custom_vars` in the `.gclient`
+file:
+
+```python
+solutions = [
+  {
+    "custom_vars": {
+      "reapi_address": "remotebuild.example.com:443",
+      "reapi_backend_config_path": "/path/to/backend.star",
+      "reapi_instance": "default",
+    },
+  }
+]
+```
+
+The `reapi_backend_config_path` is required and specifies which backend
+configuration to use. This Starlark file defines platform properties like the
+Docker image to use for remote workers. We currently use `google.star` which
+comes with upstream Chromium. This allows us to:
+
+- Use the same remote execution Docker image as Chromium
+- Automatically get updates when Chromium updates their toolchain requirements
+- Avoid maintaining our own backend configuration
+
+During `npm run sync`, we generate the `.gclient` file and set these
+`custom_vars` based on the `rbe_service` variable in `brave/.env`. This results
+in a `.sisoenv` file like:
+
+```bash
+SISO_REAPI_INSTANCE=default
+SISO_REAPI_ADDRESS=remotebuild.example.com:443
+```
+
+This file tells Siso:
+
+- **Where to connect** for remote execution (the cluster address)
+- **Which instance to use** on that cluster
+
+Without this configuration, Siso would try to use Chromium's default remote
+execution service (Google's RBE).
+
+### `.sisorc` - Ninja/Siso Runtime Flags
+
+This file is generated by `brave_custom.py` in the `configure_sisorc()` function
+during `gclient runhooks` (via `npm run sync`). It configures Siso's behavior
+for the `ninja` subcommand:
+
+```bash
+ninja -reapi_keep_exec_stream -cache_dir "/path/to/cache" -local_cache_enable
+```
+
+This file configures Siso's behavior for `ninja` subcommand:
+
+- **`-reapi_keep_exec_stream`**: Keeps the connection to the remote execution
+  service alive continuously. Our backend requires this; without it, Siso would
+  disconnect every minute, aborting long-running remote compilations.
+
+- **`-cache_dir` and `-local_cache_enable`**: Enables a local disk cache for
+  remote execution results if the `SISO_CACHE_DIR` environment variable is set.
+  The cache directory is created automatically if it doesn't exist.
+
+The reason to pass these flags via `.sisorc` instead of `npm run build` is to
+allow `autoninja` to work directly.
+
+## Architecture Diagram
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│  Developer runs: npm run sync                                   │
+│    └─> generates .gclient with custom_vars                      │
+│          ├─> reapi_address (from rbe_service .env var)          │
+│          └─> reapi_backend_config_path ('google.star')          │
+│          ├─> reapi_instance ('default')                         │
+└──────────────────────┬──────────────────────────────────────────┘
+                       │
+                       ▼
+┌─────────────────────────────────────────────────────────────────┐
+│  gclient runhooks                                               │
+│    ├─> configure_siso.py (upstream Chromium)                    │
+│    │     ├─> write .sisoenv (from .gclient custom_vars)         │
+│    │     └─> copy /path/to/backend.star → backend.star          │
+│    └─> reclient configurator                                    │
+│          └─> brave_custom.py callbacks                          │
+│                ├─> generate python_remote_wrapper               │
+│                └─> write .sisorc (ninja flags from env vars)    │
+└──────────────────────┬──────────────────────────────────────────┘
+                       │
+                       ▼
+┌─────────────────────────────────────────────────────────────────┐
+│  Developer runs: ninja -C out/Component                         │
+│    └─> siso                                                     │
+│          ├─> reads .sisoenv (RBE cluster and instance config)   │
+│          ├─> reads .sisorc (runtime flags)                      │
+│          └─> loads main.star                                    │
+│                └─> calls brave_siso_config.configure()          │
+│                      ├─> adjusts step_config (rules/timeouts)   │
+│                      ├─> adjusts filegroups (exclusions)        │
+│                      └─> installs handlers                      │
+│                            ├─> redirect_cc (C++ overrides)      │
+│                            └─> chromium_src_inputs (Py actions) │
+└──────────────────────┬──────────────────────────────────────────┘
+                       │
+                       ▼
+┌─────────────────────────────────────────────────────────────────┐
+│  For each build action:                                         │
+│    ├─> Handler checks for brave/chromium_src/ overrides         │
+│    ├─> Adds overrides to inputs                                 │
+│    ├─> Uploads inputs to remote worker                          │
+│    ├─> Executes remotely (or locally if disabled)               │
+│    └─> Caches results locally and remotely                      │
+└─────────────────────────────────────────────────────────────────┘
+```
@@ -1,5 +1,5 @@
 diff --git a/build/config/siso/main.star b/build/config/siso/main.star
-index 34bb20173df31840d95ff57bafd4efb63b3d5010..1b828a0e67a597efaa77e65c98c833034556a858 100644
+index 34bb20173df31840d95ff57bafd4efb63b3d5010..567ae154c4a42e44ee8ec2a947e2756b885d06da 100644
 --- a/build/config/siso/main.star
 +++ b/build/config/siso/main.star
@@ -20,6 +20,7 @@ load("./reproxy.star", "reproxy")
@@ -14,7 +14,7 @@ index 34bb20173df31840d95ff57bafd4efb63b3d5010..1b828a0e67a597efaa77e65c98c83303
     handlers.update(simple.handlers)
     handlers.update(reproxy.handlers)
 
-+    brave_siso_config.configure(ctx, step_config, handlers)
+    brave_siso_config.configure(ctx, step_config, filegroups, handlers)
     return module(
         "config",
         step_config = json.encode(step_config),
@@ -66,6 +66,7 @@ def merge_rewrapper_large_cfg(rewrapper_cfg, tool, host_os):

 def post_configure():
    generate_python_remote_wrapper()
+    configure_sisorc()


 # Python remote wrapper sets PYTHONPATH during remote execution.
@@ -99,4 +100,24 @@ def generate_python_remote_wrapper():
    FileUtils.write_text_file(
        (f'{Paths.src_dir}/buildtools/reclient_cfgs/python/'
         'python_remote_wrapper'),
-        ShellTemplate(python_remote_wrapper_template).substitute(template_vars))
+        ShellTemplate(python_remote_wrapper_template).substitute(
+            template_vars))
+
+
+def configure_sisorc():
+    ninja_flags = []
+    if os.environ.get('RBE_service'):
+        # `-reapi_keep_exec_stream` to keep exec stream alive during remote
+        # execution, otherwise siso terminates it each minute which aborts
+        # remote exec in EngFlow backend.
+        ninja_flags.append('-reapi_keep_exec_stream')
+
+    if cache_dir := os.environ.get('SISO_CACHE_DIR'):
+        # `-cache_dir` and `-local_cache_enable` to use a local disk cache for
+        # remote execution. Cache is disabled if `SISO_CACHE_DIR` is not set.
+        ninja_flags.append(f'-cache_dir "{cache_dir}" -local_cache_enable')
+        os.makedirs(cache_dir, exist_ok=True)
+
+    FileUtils.write_text_file(
+        f'{Paths.src_dir}/build/config/siso/.sisorc',
+        f'ninja {" ".join(ninja_flags)}\n' if ninja_flags else '')