diff --git a/tools/cr/toolchain/README.md b/tools/cr/toolchain/README.md index a3a22931b73..929daf037ce 100644 --- a/tools/cr/toolchain/README.md +++ b/tools/cr/toolchain/README.md @@ -20,6 +20,23 @@ curl -sL \ ## Scripts +### `bootstrap_depot_tools.py` + +Bootstraps `depot_tools` on a fresh CI worker, and then runs a given python +script using `vpython3`. This allows for the toolchain python scripts to be run +across different environments with the same guarantees. Arguments after `--` +are forwarded verbatim. + +```sh +curl -sSLf \ + https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/bootstrap_depot_tools.py \ + | python3 - \ + --run=https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/build_rust_toolchain.py \ + -- \ + --out-dir=./out/ --chromium-src=chromium/src --clone-chromium \ + --use-ref=150.0.7850.1 +``` + ### `build_rust_toolchain.py` Builds and packages a minimal Rust toolchain subset for Chromium: the diff --git a/tools/cr/toolchain/bootstrap_depot_tools.py b/tools/cr/toolchain/bootstrap_depot_tools.py new file mode 100755 index 00000000000..45ce50c4b83 --- /dev/null +++ b/tools/cr/toolchain/bootstrap_depot_tools.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 The Brave Authors. All rights reserved. +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. +"""Bootstrap depot_tools and forward to a Python script under vpython3. + +Designed for easy deployment of vpython3, specially in CI environments. +The script: + + 1. Locates or clones `depot_tools`. + 2. Prepends it to `PATH` and triggers its self-bootstrap so that `vpython3` + and its on-demand resources are ready. + 3. Resolves the target Python script named by `--run`: an `http(s)` + URL is downloaded into a temp directory, while a local filesystem + path is used in place. + 4. Invokes it under `depot_tools/vpython3`, forwarding any arguments that + follow `--` verbatim, and propagates its exit code. + +Keep this script standalone, single file, Python standard library only script, +so it can run straight from `curl | python3 -` on a fresh CI worker, before +any project checkout exists. + +Example: + +```sh +curl -sSLf \ + https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/bootstrap_depot_tools.py \ + | python3 - \ + --run=https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/build_rust_toolchain.py \ + -- \ + --out-dir=./out \ + --chromium-src=chromium/src \ + --clone-chromium \ + --use-ref=150.0.7850.1 +``` +""" + +from __future__ import annotations + +import argparse +import contextlib +import logging +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request +from collections.abc import Iterator +from pathlib import Path + +# Latest Chromium depot_tools bundle. +DEPOT_TOOLS_URL = 'https://chromium.googlesource.com/chromium/tools/depot_tools' + +# Name of the vpython3 launcher shipped inside depot_tools. +VPYTHON_NAME = 'vpython3.bat' if sys.platform == 'win32' else 'vpython3' + + +def _check_call(*command: str, cwd: Path | None = None) -> None: + """Run *command* synchronously, logging the invocation. + """ + logging.info(' >>>> %s', ' '.join(str(a) for a in command)) + + if platform.system() == 'Windows': + resolved = shutil.which(command[0]) + if resolved is None: + raise RuntimeError(f'Command not found: {command[0]}') + if resolved != command[0]: + command = (resolved, *command[1:]) + + subprocess.run(command, cwd=cwd, check=True) + + +def _bootstrap_depot_tools(depot_tools_dir: Path) -> Path: + """Ensure depot_tools is installed and reachable; return the install dir. + + Resolution order: + 1. If `gclient` is already on PATH, use that install location. + 2. If `depot_tools_dir` already contains a `gclient`, reuse it. + 3. Otherwise clone `depot_tools` into `depot_tools_dir`. + + The chosen install directory is prepended to PATH, then a no-op + `gclient` invocation is issued so that depot_tools can self-update and + lay down its bundled Python interpreter before the target script runs. + """ + existing = shutil.which('gclient') + if existing is not None: + install = Path(existing).resolve().parent + logging.info('depot_tools already on PATH at %s', install) + elif (depot_tools_dir / 'gclient').is_file(): + install = depot_tools_dir.resolve() + logging.info('Reusing depot_tools at %s', install) + else: + install = depot_tools_dir.resolve() + logging.info('Cloning depot_tools into %s', install) + install.parent.mkdir(parents=True, exist_ok=True) + _check_call('git', 'clone', DEPOT_TOOLS_URL, str(install)) + + os.environ['PATH'] = os.pathsep.join([str(install), os.environ['PATH']]) + + # A no-arg gclient call triggers depot_tools' self-bootstrap, which + # downloads vpython3's bundled Python and other on-demand resources. + _check_call('gclient') + + return install + + +@contextlib.contextmanager +def _resolve_script(script: str) -> Iterator[Path]: + """Yield a local Path to *script*, downloading it via HTTP(S) if needed. + + Strings with an `http` / `https` scheme are fetched into a temporary + directory that is removed when the context manager exits. Anything else + is treated as a local filesystem path and yielded in place. + """ + parsed = urllib.parse.urlparse(script) + if parsed.scheme in ('http', 'https'): + with tempfile.TemporaryDirectory( + prefix='bootstrap_depot_tools_') as tmp: + name = Path(parsed.path).name or 'script.py' + dest = Path(tmp) / name + logging.info('Downloading %s -> %s', script, dest) + with urllib.request.urlopen(script) as response: + dest.write_bytes(response.read()) + yield dest + return + + local = Path(script).expanduser().resolve() + if not local.is_file(): + raise RuntimeError(f'Script not found: {local}') + yield local + + +def _split_argv(argv: list[str]) -> tuple[list[str], list[str]]: + """Split *argv* on the first `--`. + + This bypasses argparse for forwarded arguments so that they cannot + accidentally collide with this script's own options (e.g. a forwarded + `--verbose` going to the target script instead of being claimed here). + """ + if '--' in argv: + idx = argv.index('--') + return argv[:idx], argv[idx + 1:] + return argv, [] + + +def main() -> int: + own_args, forwarded = _split_argv(sys.argv[1:]) + + parser = argparse.ArgumentParser( + description='Bootstrap depot_tools and forward to a Python script ' + 'under vpython3. Arguments after `--` are forwarded verbatim to ' + 'the target script.') + parser.add_argument( + '--run', + required=True, + help='URL (http/https) or local path to the Python script to run ' + 'under vpython3 once depot_tools is ready.') + parser.add_argument( + '--depot-tools-dir', + default='./depot_tools', + help='Directory to install depot_tools into when no existing copy ' + 'is found (default: ./depot_tools).') + parser.add_argument('--verbose', + action='store_true', + help='Enable verbose (debug) logging.') + args = parser.parse_args(own_args) + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, + format='%(message)s') + + install = _bootstrap_depot_tools(Path(args.depot_tools_dir).expanduser()) + + vpython = install / VPYTHON_NAME + if not vpython.exists(): + raise RuntimeError(f'vpython3 not found at {vpython}') + + with _resolve_script(args.run) as script_path: + cmd = (str(vpython), str(script_path), *forwarded) + logging.info(' >>>> %s', ' '.join(cmd)) + result = subprocess.run(cmd, check=False) + return result.returncode + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/cr/toolchain/build_rust_toolchain.py b/tools/cr/toolchain/build_rust_toolchain.py index 5388879655a..b7aa06cc382 100755 --- a/tools/cr/toolchain/build_rust_toolchain.py +++ b/tools/cr/toolchain/build_rust_toolchain.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env vpython3 # Copyright (c) 2026 The Brave Authors. All rights reserved. # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, @@ -74,6 +74,7 @@ import shutil import subprocess import sys import tarfile +import tomllib from types import ModuleType # Filename of the LLVM linker binary produced by the Chromium LLVM build. @@ -221,46 +222,14 @@ class ToolchainBuilder: `$LLVM_BIN` placeholders inside string values are preserved verbatim — `build_rust.py` substitutes them when it generates `config.toml`. + Bare `$PLACEHOLDER` lines (not valid TOML) are stripped before + parsing. """ target_triple = self.build_rust_module.RustTargetTriple() text = self.config_toml_template.read_bytes().decode('utf-8') - if sys.version_info >= (3, 11): - # `tomllib` is stdlib since 3.11; the Linux CI node still runs - # 3.10, hence the fallback below. Drop both this import and the - # fallback once every node is on 3.11+. - import tomllib - text = re.sub(r'(?m)^\$[A-Z_]+\s*$\n?', '', text) - data = tomllib.loads(text) - return dict(data['target'][target_triple]) - - # Python <=3.10 fallback: walk the file and collect `key = value` - # lines inside the target's section, ignoring blanks, comments and - # the `$PLACEHOLDER` lines that confuse a real TOML parser anyway. - header = f'[target.{target_triple}]' - result: dict[str, str | bool] = {} - in_section = False - for line in text.splitlines(): - stripped = line.strip() - if stripped.startswith('['): - in_section = stripped == header - continue - if (not in_section or not stripped or stripped.startswith('#') - or '=' not in stripped): - continue - key, _, raw = stripped.partition('=') - key = key.strip() - raw = raw.strip() - if raw == 'true': - result[key] = True - elif raw == 'false': - result[key] = False - elif (len(raw) >= 2 and raw[0] == raw[-1] - and raw[0] in ('"', "'")): - result[key] = raw[1:-1] - else: - raise ValueError(f'Cannot parse {key!r} = {raw!r} in {header} ' - 'fallback parser') - return result + text = re.sub(r'(?m)^\$[A-Z_]+\s*$\n?', '', text) + data = tomllib.loads(text) + return dict(data['target'][target_triple]) @staticmethod def _emit_toml_kv(key: str, value: str | bool) -> str: diff --git a/tools/cr/toolchain/build_xcode_toolchain.py b/tools/cr/toolchain/build_xcode_toolchain.py index 42e06d875ff..6889cc09a21 100755 --- a/tools/cr/toolchain/build_xcode_toolchain.py +++ b/tools/cr/toolchain/build_xcode_toolchain.py @@ -1,21 +1,42 @@ -#!/usr/bin/env python3 +#!/usr/bin/env vpython3 # Copyright (c) 2026 The Brave Authors. All rights reserved. # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at https://mozilla.org/MPL/2.0/. +# +# [VPYTHON:BEGIN] +# python_version: "3.11" +# +# wheel: < +# name: "infra/python/wheels/pyyaml-py3" +# version: "version:6.0.1" +# > +# [VPYTHON:END] """Build a hermetic, reproducible Xcode toolchain archive for Chromium. Keep this as a *standalone* script that can be invoked directly on a macOS CI -node. PyYAML is the only non-stdlib import; if it is not already importable it -is installed via `pip --user` on the first run, so a fresh CI image needs no -preconfiguration beyond `python3` and `pip`. +node. PyYAML is the only non-stdlib import and is provided by the inline +vpython3 spec above, so either use a vpython environment, or make sure to +install PyYAML into the system Python if running directly. -To run directly from GitHub: +To run directly from GitHub on a worker that already has depot_tools on PATH: ```sh curl -sL \ https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/build_xcode_toolchain.py \ + | vpython3 - \ + --out-dir=./out/ \ + --chromium-tag=150.0.7841.1 +``` + +Or via the bootstrap forwarder on a fresh worker: + +```sh +curl -sSLf \ + https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/bootstrap_depot_tools.py \ | python3 - \ + --run=https://raw.githubusercontent.com/brave/brave-core/refs/heads/master/tools/cr/toolchain/build_xcode_toolchain.py \ + -- \ --out-dir=./out/ \ --chromium-tag=150.0.7841.1 ```