From a7cb22bb671a4aa980dadcef1e8315feca2d472d Mon Sep 17 00:00:00 2001 From: cdesouza-chromium Date: Thu, 28 May 2026 14:44:40 +0100 Subject: [PATCH] [toolchain] A bootstrap for `depot_tools` (#36811) This bootstrap script will allows us to have a stable environment across all bots runnning toolchain scripts. The basics of this script is that we can chain it with the toolchain ones, and it takes care that we have a valid `vpython` install to run our tools on. It is possible to use both a url to download a script, or to provide a local one. This is being done to correct some of the issues we are having in CI with python deps, but it will also permit us to come up with subrevision schemes for toolchains that take into account the hashing of the builder. Bug: https://github.com/brave/brave-browser/issues/55812 --- tools/cr/toolchain/README.md | 17 ++ tools/cr/toolchain/bootstrap_depot_tools.py | 189 ++++++++++++++++++++ tools/cr/toolchain/build_rust_toolchain.py | 45 +---- tools/cr/toolchain/build_xcode_toolchain.py | 31 +++- 4 files changed, 239 insertions(+), 43 deletions(-) create mode 100755 tools/cr/toolchain/bootstrap_depot_tools.py diff --git a/tools/cr/toolchain/README.md b/tools/cr/toolchain/README.md index a3a22931b73..929daf037ce 100644 --- a/tools/cr/toolchain/README.md +++ b/tools/cr/toolchain/README.md @@ -20,6 +20,23 @@ curl -sL \ ## Scripts +### `bootstrap_depot_tools.py` + +Bootstraps `depot_tools` on a fresh CI worker, and then runs a given python +script using `vpython3`. This allows for the toolchain python scripts to be run +across different environments with the same guarantees. Arguments after `--` +are forwarded verbatim. + +```sh +curl -sSLf \ + https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/bootstrap_depot_tools.py \ + | python3 - \ + --run=https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/build_rust_toolchain.py \ + -- \ + --out-dir=./out/ --chromium-src=chromium/src --clone-chromium \ + --use-ref=150.0.7850.1 +``` + ### `build_rust_toolchain.py` Builds and packages a minimal Rust toolchain subset for Chromium: the diff --git a/tools/cr/toolchain/bootstrap_depot_tools.py b/tools/cr/toolchain/bootstrap_depot_tools.py new file mode 100755 index 00000000000..45ce50c4b83 --- /dev/null +++ b/tools/cr/toolchain/bootstrap_depot_tools.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 The Brave Authors. All rights reserved. +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. +"""Bootstrap depot_tools and forward to a Python script under vpython3. + +Designed for easy deployment of vpython3, specially in CI environments. +The script: + + 1. Locates or clones `depot_tools`. + 2. Prepends it to `PATH` and triggers its self-bootstrap so that `vpython3` + and its on-demand resources are ready. + 3. Resolves the target Python script named by `--run`: an `http(s)` + URL is downloaded into a temp directory, while a local filesystem + path is used in place. + 4. Invokes it under `depot_tools/vpython3`, forwarding any arguments that + follow `--` verbatim, and propagates its exit code. + +Keep this script standalone, single file, Python standard library only script, +so it can run straight from `curl | python3 -` on a fresh CI worker, before +any project checkout exists. + +Example: + +```sh +curl -sSLf \ + https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/bootstrap_depot_tools.py \ + | python3 - \ + --run=https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/build_rust_toolchain.py \ + -- \ + --out-dir=./out \ + --chromium-src=chromium/src \ + --clone-chromium \ + --use-ref=150.0.7850.1 +``` +""" + +from __future__ import annotations + +import argparse +import contextlib +import logging +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request +from collections.abc import Iterator +from pathlib import Path + +# Latest Chromium depot_tools bundle. +DEPOT_TOOLS_URL = 'https://chromium.googlesource.com/chromium/tools/depot_tools' + +# Name of the vpython3 launcher shipped inside depot_tools. +VPYTHON_NAME = 'vpython3.bat' if sys.platform == 'win32' else 'vpython3' + + +def _check_call(*command: str, cwd: Path | None = None) -> None: + """Run *command* synchronously, logging the invocation. + """ + logging.info(' >>>> %s', ' '.join(str(a) for a in command)) + + if platform.system() == 'Windows': + resolved = shutil.which(command[0]) + if resolved is None: + raise RuntimeError(f'Command not found: {command[0]}') + if resolved != command[0]: + command = (resolved, *command[1:]) + + subprocess.run(command, cwd=cwd, check=True) + + +def _bootstrap_depot_tools(depot_tools_dir: Path) -> Path: + """Ensure depot_tools is installed and reachable; return the install dir. + + Resolution order: + 1. If `gclient` is already on PATH, use that install location. + 2. If `depot_tools_dir` already contains a `gclient`, reuse it. + 3. Otherwise clone `depot_tools` into `depot_tools_dir`. + + The chosen install directory is prepended to PATH, then a no-op + `gclient` invocation is issued so that depot_tools can self-update and + lay down its bundled Python interpreter before the target script runs. + """ + existing = shutil.which('gclient') + if existing is not None: + install = Path(existing).resolve().parent + logging.info('depot_tools already on PATH at %s', install) + elif (depot_tools_dir / 'gclient').is_file(): + install = depot_tools_dir.resolve() + logging.info('Reusing depot_tools at %s', install) + else: + install = depot_tools_dir.resolve() + logging.info('Cloning depot_tools into %s', install) + install.parent.mkdir(parents=True, exist_ok=True) + _check_call('git', 'clone', DEPOT_TOOLS_URL, str(install)) + + os.environ['PATH'] = os.pathsep.join([str(install), os.environ['PATH']]) + + # A no-arg gclient call triggers depot_tools' self-bootstrap, which + # downloads vpython3's bundled Python and other on-demand resources. + _check_call('gclient') + + return install + + +@contextlib.contextmanager +def _resolve_script(script: str) -> Iterator[Path]: + """Yield a local Path to *script*, downloading it via HTTP(S) if needed. + + Strings with an `http` / `https` scheme are fetched into a temporary + directory that is removed when the context manager exits. Anything else + is treated as a local filesystem path and yielded in place. + """ + parsed = urllib.parse.urlparse(script) + if parsed.scheme in ('http', 'https'): + with tempfile.TemporaryDirectory( + prefix='bootstrap_depot_tools_') as tmp: + name = Path(parsed.path).name or 'script.py' + dest = Path(tmp) / name + logging.info('Downloading %s -> %s', script, dest) + with urllib.request.urlopen(script) as response: + dest.write_bytes(response.read()) + yield dest + return + + local = Path(script).expanduser().resolve() + if not local.is_file(): + raise RuntimeError(f'Script not found: {local}') + yield local + + +def _split_argv(argv: list[str]) -> tuple[list[str], list[str]]: + """Split *argv* on the first `--`. + + This bypasses argparse for forwarded arguments so that they cannot + accidentally collide with this script's own options (e.g. a forwarded + `--verbose` going to the target script instead of being claimed here). + """ + if '--' in argv: + idx = argv.index('--') + return argv[:idx], argv[idx + 1:] + return argv, [] + + +def main() -> int: + own_args, forwarded = _split_argv(sys.argv[1:]) + + parser = argparse.ArgumentParser( + description='Bootstrap depot_tools and forward to a Python script ' + 'under vpython3. Arguments after `--` are forwarded verbatim to ' + 'the target script.') + parser.add_argument( + '--run', + required=True, + help='URL (http/https) or local path to the Python script to run ' + 'under vpython3 once depot_tools is ready.') + parser.add_argument( + '--depot-tools-dir', + default='./depot_tools', + help='Directory to install depot_tools into when no existing copy ' + 'is found (default: ./depot_tools).') + parser.add_argument('--verbose', + action='store_true', + help='Enable verbose (debug) logging.') + args = parser.parse_args(own_args) + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, + format='%(message)s') + + install = _bootstrap_depot_tools(Path(args.depot_tools_dir).expanduser()) + + vpython = install / VPYTHON_NAME + if not vpython.exists(): + raise RuntimeError(f'vpython3 not found at {vpython}') + + with _resolve_script(args.run) as script_path: + cmd = (str(vpython), str(script_path), *forwarded) + logging.info(' >>>> %s', ' '.join(cmd)) + result = subprocess.run(cmd, check=False) + return result.returncode + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/cr/toolchain/build_rust_toolchain.py b/tools/cr/toolchain/build_rust_toolchain.py index 5388879655a..b7aa06cc382 100755 --- a/tools/cr/toolchain/build_rust_toolchain.py +++ b/tools/cr/toolchain/build_rust_toolchain.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env vpython3 # Copyright (c) 2026 The Brave Authors. All rights reserved. # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, @@ -74,6 +74,7 @@ import shutil import subprocess import sys import tarfile +import tomllib from types import ModuleType # Filename of the LLVM linker binary produced by the Chromium LLVM build. @@ -221,46 +222,14 @@ class ToolchainBuilder: `$LLVM_BIN` placeholders inside string values are preserved verbatim — `build_rust.py` substitutes them when it generates `config.toml`. + Bare `$PLACEHOLDER` lines (not valid TOML) are stripped before + parsing. """ target_triple = self.build_rust_module.RustTargetTriple() text = self.config_toml_template.read_bytes().decode('utf-8') - if sys.version_info >= (3, 11): - # `tomllib` is stdlib since 3.11; the Linux CI node still runs - # 3.10, hence the fallback below. Drop both this import and the - # fallback once every node is on 3.11+. - import tomllib - text = re.sub(r'(?m)^\$[A-Z_]+\s*$\n?', '', text) - data = tomllib.loads(text) - return dict(data['target'][target_triple]) - - # Python <=3.10 fallback: walk the file and collect `key = value` - # lines inside the target's section, ignoring blanks, comments and - # the `$PLACEHOLDER` lines that confuse a real TOML parser anyway. - header = f'[target.{target_triple}]' - result: dict[str, str | bool] = {} - in_section = False - for line in text.splitlines(): - stripped = line.strip() - if stripped.startswith('['): - in_section = stripped == header - continue - if (not in_section or not stripped or stripped.startswith('#') - or '=' not in stripped): - continue - key, _, raw = stripped.partition('=') - key = key.strip() - raw = raw.strip() - if raw == 'true': - result[key] = True - elif raw == 'false': - result[key] = False - elif (len(raw) >= 2 and raw[0] == raw[-1] - and raw[0] in ('"', "'")): - result[key] = raw[1:-1] - else: - raise ValueError(f'Cannot parse {key!r} = {raw!r} in {header} ' - 'fallback parser') - return result + text = re.sub(r'(?m)^\$[A-Z_]+\s*$\n?', '', text) + data = tomllib.loads(text) + return dict(data['target'][target_triple]) @staticmethod def _emit_toml_kv(key: str, value: str | bool) -> str: diff --git a/tools/cr/toolchain/build_xcode_toolchain.py b/tools/cr/toolchain/build_xcode_toolchain.py index 42e06d875ff..6889cc09a21 100755 --- a/tools/cr/toolchain/build_xcode_toolchain.py +++ b/tools/cr/toolchain/build_xcode_toolchain.py @@ -1,21 +1,42 @@ -#!/usr/bin/env python3 +#!/usr/bin/env vpython3 # Copyright (c) 2026 The Brave Authors. All rights reserved. # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at https://mozilla.org/MPL/2.0/. +# +# [VPYTHON:BEGIN] +# python_version: "3.11" +# +# wheel: < +# name: "infra/python/wheels/pyyaml-py3" +# version: "version:6.0.1" +# > +# [VPYTHON:END] """Build a hermetic, reproducible Xcode toolchain archive for Chromium. Keep this as a *standalone* script that can be invoked directly on a macOS CI -node. PyYAML is the only non-stdlib import; if it is not already importable it -is installed via `pip --user` on the first run, so a fresh CI image needs no -preconfiguration beyond `python3` and `pip`. +node. PyYAML is the only non-stdlib import and is provided by the inline +vpython3 spec above, so either use a vpython environment, or make sure to +install PyYAML into the system Python if running directly. -To run directly from GitHub: +To run directly from GitHub on a worker that already has depot_tools on PATH: ```sh curl -sL \ https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/build_xcode_toolchain.py \ + | vpython3 - \ + --out-dir=./out/ \ + --chromium-tag=150.0.7841.1 +``` + +Or via the bootstrap forwarder on a fresh worker: + +```sh +curl -sSLf \ + https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/bootstrap_depot_tools.py \ | python3 - \ + --run=https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/build_xcode_toolchain.py \ + -- \ --out-dir=./out/ \ --chromium-tag=150.0.7841.1 ```