From 05ac071ab54571108b8d7d87fba0d521bc0ce716 Mon Sep 17 00:00:00 2001 From: Aleksey Khoroshilov Date: Mon, 20 Nov 2023 13:43:21 +0700 Subject: [PATCH] Replace reclient configurator with EngFlow-supported one. --- .gitignore | 1 + DEPS | 6 +- build/reclient_cfgs/README.md | 64 ---- .../clang_remote_wrapper.template | 42 --- .../chromium-browser-clang/rewrapper_base.cfg | 10 - .../rewrapper_linux.cfg | 7 - .../chromium-browser-clang/rewrapper_mac.cfg | 11 - .../rewrapper_windows.cfg | 11 - build/reclient_cfgs/configure_reclient.py | 356 ------------------ build/reclient_cfgs/python/rewrapper_base.cfg | 12 - .../reclient_cfgs/python/rewrapper_linux.cfg | 7 - build/reclient_cfgs/python/rewrapper_mac.cfg | 7 - .../python/rewrapper_windows.cfg | 7 - build/reclient_cfgs/reproxy.cfg | 14 - script/brave_license_helper.py | 6 + .../brave_custom/brave_custom.py | 112 ++++++ .../python_remote_wrapper.template | 5 +- 17 files changed, 127 insertions(+), 551 deletions(-) delete mode 100644 build/reclient_cfgs/README.md delete mode 100755 build/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper.template delete mode 100644 build/reclient_cfgs/chromium-browser-clang/rewrapper_base.cfg delete mode 100644 build/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg delete mode 100644 build/reclient_cfgs/chromium-browser-clang/rewrapper_mac.cfg delete mode 100644 build/reclient_cfgs/chromium-browser-clang/rewrapper_windows.cfg delete mode 100755 build/reclient_cfgs/configure_reclient.py delete mode 100644 build/reclient_cfgs/python/rewrapper_base.cfg delete mode 100644 build/reclient_cfgs/python/rewrapper_linux.cfg delete mode 100644 build/reclient_cfgs/python/rewrapper_mac.cfg delete mode 100644 build/reclient_cfgs/python/rewrapper_windows.cfg delete mode 100644 build/reclient_cfgs/reproxy.cfg create mode 100644 third_party/reclient_configs/brave_custom/brave_custom.py rename {build/reclient_cfgs/python => third_party/reclient_configs/brave_custom}/python_remote_wrapper.template (90%) diff --git a/.gitignore b/.gitignore index b177f377619..fd13b56cb17 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ patches/**/*.patchinfo /third_party/bitcoin-core/src /third_party/playlist_component/src /third_party/rapidjson/src +/third_party/reclient_configs/src /third_party/rust/challenge_bypass_ristretto/v1/crate /third_party/rust/kuchiki/v0_8/crate /third_party/rust/futures_retry/v0_5/crate diff --git a/DEPS b/DEPS index 422511ab61d..a200ddf1d1f 100644 --- a/DEPS +++ b/DEPS @@ -24,6 +24,7 @@ deps = { "third_party/bitcoin-core/src": "https://github.com/bitcoin/bitcoin.git@8105bce5b384c72cf08b25b7c5343622754e7337", # v25.0 "third_party/argon2/src": "https://github.com/P-H-C/phc-winner-argon2.git@62358ba2123abd17fccf2a108a301d4b52c01a7c", "third_party/rapidjson/src": "https://github.com/Tencent/rapidjson.git@06d58b9e848c650114556a23294d0b6440078c61", + "third_party/reclient_configs/src": "https://github.com/EngFlow/reclient-configs.git@61ddd9705962a6c6c921b42c933451d4810aa28e", 'third_party/android_deps/libs/com_google_android_play_core': { 'packages': [ { @@ -65,7 +66,10 @@ hooks = [ { 'name': 'configure_reclient', 'pattern': '.', - 'action': ['python3', 'build/reclient_cfgs/configure_reclient.py'], + 'action': ['python3', 'third_party/reclient_configs/src/configure_reclient.py', + '--src_dir=..', + '--exec_root=../..', + '--custom_py=third_party/reclient_configs/brave_custom/brave_custom.py'], }, { 'name': 'download_sparkle', diff --git a/build/reclient_cfgs/README.md b/build/reclient_cfgs/README.md deleted file mode 100644 index 8a7ef056f4e..00000000000 --- a/build/reclient_cfgs/README.md +++ /dev/null @@ -1,64 +0,0 @@ -This directory contains configuration script and reclient configs to support -compilation on linux remotes. - -### Configuration steps of `configure_reclient.py` - -1. Generate reproxy and rewrapper configs. -2. Generate `python_remote_wrapper`. -3. If a host machine is not linux, download linux toolchain and generate - `clang_remote_wrapper`. - -##### Reproxy config merge - -Reproxy config is based on the Chromium config template. -1. Load and substitute - `//buildtools/reclient_cfgs/reproxy_cfg_templates/reproxy.cfg.template` with - empty variables. This is done to detect new template variables in the config. -2. Merge `//brave/build/reclient_cfgs/reproxy.cfg`. -3. Set auth-specific vars from `RBE_*` environment variables if they are set. - -##### Rewrapper configs merge - -Rewrapper configs are based on the Chromium linux config. -1. Load `//buildtools/reclient_cfgs/linux//rewrapper_linux.cfg` -2. Merge `//brave/build/reclient_cfgs//rewrapper_base.cfg` -3. Merge `//brave/build/reclient_cfgs//rewrapper_.cfg` - -##### Remote wrappers - -`python_remote_wrapper` adds `PYTHONPATH` into envrionment and runs the passed -command as is. - -`clang_remote_wrapper` is required to run cross-compilation. It replaces default -clang path with a linux clang path and runs the linux version of the clang -remotely. - -### Config merge process - -1. Parse a config item and merge it with the existing one or create a new one. -2. If the value is a map, overwrite map items; if it's a list, append - list items. Map-like and list-like items are hardcoded in - `from_reclient_cfg_value()`. -3. If the value is empty, clear the item. - -This allows config merger to perform such modifications: - -``` -# Add/modify/remove items into a map-like value. -# This adds/modifies "OSFamily" and removes "label:action_default". -platform=OSFamily=linux,label:action_default= - -# Add items into a list-like value. -inputs=src/new_file,src/new_file2 - -# Modify a simple value. -canonicalize_working_dir=false - -# Fully replace list/map values by clearing it first. -labels= -labels=type=compile,compiler=clang,lang=cpp -platform= -platform=container-image=docker://... -inputs= -inputs=src/a_single_input -``` diff --git a/build/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper.template b/build/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper.template deleted file mode 100755 index 01d3f580a87..00000000000 --- a/build/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper.template +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -%{AUTOGENERATED_HEADER} -# WARNING: This file is a part of reclient action inputs. Any modification will -# invalidate remote cache. - -set -e - -CLANG_DIR="%{CLANG_DIR}" -LINUX_CLANG_DIR="%{LINUX_CLANG_DIR}" -CLANG_INCLUDE_DIR="%{CLANG_INCLUDE_DIR}" -LINUX_CLANG_INCLUDE_DIR="%{LINUX_CLANG_INCLUDE_DIR}" - -# Fix PATH variable when it's passed from a Windows host. -PATH="${PATH//\\//}" -export PATH="${PATH//;/:}" - -# Symlink clang binary to clang++ and clang-cl. -for clang_symlink in clang++ clang-cl; do - if [[ ! -f $LINUX_CLANG_DIR/bin/$clang_symlink ]]; then - ln -sr "$LINUX_CLANG_DIR/bin/clang" "$LINUX_CLANG_DIR/bin/$clang_symlink" - fi -done - -# Symlink "include" directory from CLANG_INCLUDE_DIR to LINUX_CLANG_INCLUDE_DIR. -if [[ -d $CLANG_INCLUDE_DIR && ! -d $LINUX_CLANG_INCLUDE_DIR ]]; then - mkdir -p "$(dirname "${LINUX_CLANG_INCLUDE_DIR}")" - ln -sr "$CLANG_INCLUDE_DIR" "$LINUX_CLANG_INCLUDE_DIR" -fi - -# Convert clang binary path to the linux version. -linux_clang_binary="$1" -linux_clang_binary="${linux_clang_binary//\\//}" -linux_clang_binary="${linux_clang_binary/"$CLANG_DIR"/"$LINUX_CLANG_DIR"}" -linux_clang_binary="${linux_clang_binary/.exe/}" - -# Launch linux clang binary. -"$linux_clang_binary" "${@:2}" diff --git a/build/reclient_cfgs/chromium-browser-clang/rewrapper_base.cfg b/build/reclient_cfgs/chromium-browser-clang/rewrapper_base.cfg deleted file mode 100644 index a9e838ecf97..00000000000 --- a/build/reclient_cfgs/chromium-browser-clang/rewrapper_base.cfg +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config. See -# //brave/build/reclient_cfgs/README.md. - -platform=OSFamily=linux,label:action_default= -canonicalize_working_dir=false diff --git a/build/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg b/build/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg deleted file mode 100644 index c1eb42123a4..00000000000 --- a/build/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config. See -# //brave/build/reclient_cfgs/README.md. diff --git a/build/reclient_cfgs/chromium-browser-clang/rewrapper_mac.cfg b/build/reclient_cfgs/chromium-browser-clang/rewrapper_mac.cfg deleted file mode 100644 index de0c650ae15..00000000000 --- a/build/reclient_cfgs/chromium-browser-clang/rewrapper_mac.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config. See -# //brave/build/reclient_cfgs/README.md. - -remote_wrapper=../../buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper -inputs=src/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper -toolchain_inputs=src/third_party/llvm-build/Release+Asserts_linux/bin/clang diff --git a/build/reclient_cfgs/chromium-browser-clang/rewrapper_windows.cfg b/build/reclient_cfgs/chromium-browser-clang/rewrapper_windows.cfg deleted file mode 100644 index de0c650ae15..00000000000 --- a/build/reclient_cfgs/chromium-browser-clang/rewrapper_windows.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config. See -# //brave/build/reclient_cfgs/README.md. - -remote_wrapper=../../buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper -inputs=src/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper -toolchain_inputs=src/third_party/llvm-build/Release+Asserts_linux/bin/clang diff --git a/build/reclient_cfgs/configure_reclient.py b/build/reclient_cfgs/configure_reclient.py deleted file mode 100755 index 3b4da07e599..00000000000 --- a/build/reclient_cfgs/configure_reclient.py +++ /dev/null @@ -1,356 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -import glob -import os -import re -import shutil -import string -import subprocess -import sys - -SCRIPT_DIR = os.path.dirname(__file__).replace('\\', '/') -SRC_DIR = os.path.abspath(f'{SCRIPT_DIR}/../../..').replace('\\', '/') -EXEC_ROOT = os.path.abspath(f'{SRC_DIR}/..').replace('\\', '/') - -BUILD_DIR = f'{SRC_DIR}/out/a' # Used to generate relative paths. -RECLIENT_CFGS_DIR = f'{SRC_DIR}/buildtools/reclient_cfgs' -CLANG_DIR = f'{SRC_DIR}/third_party/llvm-build/Release+Asserts' -LINUX_CLANG_DIR = f'{SRC_DIR}/third_party/llvm-build/Release+Asserts_linux' - -GENERATED_FILE_HEADER = '''# AUTOGENERATED FILE - DO NOT EDIT -# Generated by: -{source_script} -# -# To edit update: -{source_files} -# And run 'npm run sync'. -''' - - -class ShellTemplate(string.Template): - delimiter = '%' - - -def main(): - if not os.environ.get('RBE_service'): - # Do nothing if RBE environment is not configured. - return - - # Reproxy config includes auth, and network-related parameters. - generate_reproxy_cfg() - # Rewrapper configs describe how different tools should be run remotely. - generate_rewrapper_cfgs() - # Python remote wrapper sets environment variables during remote execution. - generate_python_remote_wrapper() - - if not sys.platform.startswith('linux'): - # Linux clang toolchain and clang remote wrapper are required on - # non-linux hosts to perform cross-compilation remotely. - download_linux_clang_toolchain() - generate_clang_remote_wrapper() - - -def generate_reproxy_cfg(): - # Set Chromium config template variables to empty values. - CHROMIUM_TEMPLATE_MAPPINGS = { - 'auth_flags': '', - 'depsscanner_address': '', - 'rbe_instance': '', - 'rbe_project': '', - } - # Load Chromium config template. - reproxy_cfg = parse_reclient_cfg( - string.Template( - read_text_file( - f'{RECLIENT_CFGS_DIR}/reproxy_cfg_templates/' - 'reproxy.cfg.template')).substitute(CHROMIUM_TEMPLATE_MAPPINGS)) - # Merge with our config. - source_cfg_path = f'{SCRIPT_DIR}/reproxy.cfg' - reproxy_cfg = merge_reclient_cfg(source_cfg_path, reproxy_cfg) - - # Use scandeps_server. - depsscanner_address = (f'exec://{SRC_DIR}/' - 'buildtools/reclient/scandeps_server') - if sys.platform.startswith('win'): - depsscanner_address += '.exe' - reproxy_cfg['depsscanner_address'] = depsscanner_address - - # Set values from supoprted RBE_ environment variables. - SUPPORTED_REPROXY_ENV_VARS = ( - 'RBE_service', - 'RBE_service_no_auth', - 'RBE_tls_client_auth_cert', - 'RBE_tls_client_auth_key', - 'RBE_use_application_default_credentials', - ) - for env_var in SUPPORTED_REPROXY_ENV_VARS: - value = os.environ.get(env_var) - if value: - key = env_var[4:] - reproxy_cfg[key] = from_reclient_cfg_value(key, value) - - # Write the final config to the expected location. - write_reclient_cfg(f'{RECLIENT_CFGS_DIR}/reproxy.cfg', reproxy_cfg, - source_cfg_path) - - -def generate_rewrapper_cfgs(): - # Generate chromium-browser-clang configs. - generate_rewrapper_cfg('chromium-browser-clang', 'rewrapper_linux.cfg') - generate_rewrapper_cfg('chromium-browser-clang', 'rewrapper_mac.cfg') - generate_rewrapper_cfg('chromium-browser-clang', 'rewrapper_windows.cfg') - - # Generate python configs. - generate_rewrapper_cfg('python', 'rewrapper_linux.cfg') - generate_rewrapper_cfg('python', 'rewrapper_mac.cfg') - generate_rewrapper_cfg('python', 'rewrapper_windows.cfg') - - -def generate_rewrapper_cfg(tool_dir, cfg): - # Load Chromium config for linux remote. - rewrapper_cfg = read_reclient_cfg(f'{RECLIENT_CFGS_DIR}/linux/{tool_dir}/' - 'rewrapper_linux.cfg') - # Merge with our configs (rewrapper_base and rewrapper_). - source_cfg_paths = ( - f'{SCRIPT_DIR}/{tool_dir}/rewrapper_base.cfg', - f'{SCRIPT_DIR}/{tool_dir}/{cfg}', - ) - for source_cfg_path in source_cfg_paths: - rewrapper_cfg = merge_reclient_cfg(source_cfg_path, rewrapper_cfg) - # Write the final config to the expected location. - write_reclient_cfg(f'{RECLIENT_CFGS_DIR}/{tool_dir}/{cfg}', rewrapper_cfg, - source_cfg_paths) - - -def generate_python_remote_wrapper(): - # Load python remote wrapper template. - source_file = f'{SCRIPT_DIR}/python/python_remote_wrapper.template' - python_remote_wrapper_template = read_text_file(source_file) - - # Generate PYTHONPATH env variable with paths accessible from EXEC_ROOT. - relative_python_paths = [] - for python_path in os.environ.get('PYTHONPATH', '').split(os.pathsep): - if not python_path: - continue - abs_python_path = os.path.abspath(python_path).replace('\\', '/') - if abs_python_path.startswith(EXEC_ROOT): - relative_python_paths.append(relpath(abs_python_path, BUILD_DIR)) - - # Variables to replace in the template. - template_vars = { - 'AUTOGENERATED_HEADER': create_generated_header(source_file), - 'PYTHONPATH': ':'.join(relative_python_paths), - } - - if not template_vars['PYTHONPATH']: - print('WARNING: PYTHONPATH is empty. Remote python actions will fail.') - - # Write the python remote wrapper. - write_text_file( - f'{SRC_DIR}/buildtools/reclient_cfgs/python/python_remote_wrapper', - ShellTemplate(python_remote_wrapper_template).substitute(template_vars)) - - -def download_linux_clang_toolchain(): - subprocess.check_call([ - 'python3', - f'{SRC_DIR}/tools/clang/scripts/update.py', - '--output-dir', - f'{LINUX_CLANG_DIR}', - '--host-os', - 'linux', - ]) - - -def generate_clang_remote_wrapper(): - if not os.path.exists(CLANG_DIR): - raise RuntimeError(f'Cannot find clang directory at {CLANG_DIR}.') - - # Load clang remote wrapper template. - source_file = (f'{SCRIPT_DIR}/chromium-browser-clang/' - 'clang_remote_wrapper.template') - clang_remote_wrapper_template = read_text_file(source_file) - - # Find "include" directory inside clang installation. This directory will be - # symlinked by remote wrapper for cross-compilation to work. The path is - # clang-version dependent, so don't hardcode it. - clang_include_dir_glob = glob.glob(f'{CLANG_DIR}/lib/**/include', - recursive=True) - if not clang_include_dir_glob: - raise RuntimeError( - f'Cannot find lib/**/include dir in {CLANG_DIR}. If clang ' - f'structure has changed, please update {__file__}.') - clang_include_dir_abs = clang_include_dir_glob[0].replace('\\', '/') - assert os.path.isdir(clang_include_dir_abs), clang_include_dir_abs - clang_include_dir = relpath(clang_include_dir_abs, BUILD_DIR) - linux_clang_include_dir = relpath( - clang_include_dir_abs.replace(CLANG_DIR, LINUX_CLANG_DIR), BUILD_DIR) - - # Variables to replace in the template. - template_vars = { - 'AUTOGENERATED_HEADER': create_generated_header(source_file), - 'CLANG_DIR': relpath(CLANG_DIR, BUILD_DIR), - 'CLANG_INCLUDE_DIR': clang_include_dir, - 'LINUX_CLANG_DIR': relpath(LINUX_CLANG_DIR, BUILD_DIR), - 'LINUX_CLANG_INCLUDE_DIR': linux_clang_include_dir, - } - - # Write the clang remote wrapper. - write_text_file( - (f'{SRC_DIR}/buildtools/reclient_cfgs/chromium-browser-clang/' - 'clang_remote_wrapper'), - ShellTemplate(clang_remote_wrapper_template).substitute(template_vars)) - - -### Reclient config manipulation helpers. ### -def read_reclient_cfg(cfg_path): - return dict(read_reclient_cfg_items(cfg_path)) - - -def read_reclient_cfg_items(cfg_path): - with open(cfg_path, 'r') as f: - yield from parse_reclient_cfg_items(f) - - -def parse_reclient_cfg(cfg_string): - return dict(parse_reclient_cfg_items(cfg_string.split('\n'))) - - -def parse_reclient_cfg_items(lines): - for line in lines: - line = line.strip() - if not re.match(r'^\w+=', line): - continue - key, value = line.split('=', 1) - yield key, from_reclient_cfg_value(key, value) - - -def write_reclient_cfg(cfg_path, cfg, source_cfg_paths): - cfg_to_write = create_generated_header(source_cfg_paths) + '\n' - for key, value in cfg.items(): - formatted_value = to_reclient_cfg_value(key, value) - if formatted_value: - cfg_to_write += f'{formatted_value}\n' - - write_text_file(cfg_path, cfg_to_write) - - -def from_reclient_cfg_value(key, value): - KEY_VALUE_PARAMS = ( - 'labels', - 'platform', - ) - LIST_PARAMS = ( - 'env_var_allowlist', - 'input_list_paths', - 'inputs', - 'output_files', - 'output_directories', - 'toolchain_inputs', - ) - - if key in KEY_VALUE_PARAMS: - ret_val = {} - for sub_kv in value.split(','): - if not sub_kv: - continue - if '=' not in sub_kv: - raise RuntimeError(f'key=value expected for key: {key}') - sub_key, sub_value = sub_kv.split('=', 1) - ret_val[sub_key] = sub_value - return ret_val - - if key in LIST_PARAMS: - if not value: - return [] - return value.split(',') - - return value - - -def to_reclient_cfg_value(key, value): - if isinstance(value, dict): - sub_keys_values = [] - for sub_key, sub_value in value.items(): - sub_keys_values.append(to_reclient_cfg_value(sub_key, sub_value)) - return to_reclient_cfg_value(key, sub_keys_values) - - if isinstance(value, list): - return to_reclient_cfg_value(key, ','.join(filter(lambda v: v, value))) - - return f'{key}={value}' if value else None - - -def merge_reclient_cfg(cfg_path, reclient_cfg): - for key, value in read_reclient_cfg_items(cfg_path): - reclient_cfg = merge_reclient_cfg_item(reclient_cfg, {key: value}) - return reclient_cfg - - -def merge_reclient_cfg_item(a, b): - if isinstance(a, dict): - assert isinstance(b, dict) - if not b: - a.clear() - else: - for key in b: - if key in a: - a[key] = merge_reclient_cfg_item(a[key], b[key]) - else: - a[key] = b[key] - elif isinstance(a, list): - assert isinstance(b, list) - if not b: - a.clear() - else: - a.extend(b) - else: - a = b - - return a - - -### File helpers. ### -def read_text_file(filepath): - with open(filepath, 'r') as f: - return f.read() - - -def write_text_file(filepath, data_to_write): - if os.path.isfile(filepath): - with open(filepath, 'r') as f: - if f.read() == data_to_write: - return - - os.makedirs(os.path.dirname(filepath), exist_ok=True) - - filepath_new = filepath + '.new' - with open(filepath_new, 'w', newline='\n') as f: - f.write(data_to_write) - - shutil.move(filepath_new, filepath) - - -def create_generated_header(source_files): - if not isinstance(source_files, (list, tuple)): - source_files = (source_files, ) - source_script = f'# //{relpath(__file__, SRC_DIR)}' - source_files = '\n'.join( - [f'# //{relpath(f, SRC_DIR)}' for f in source_files]) - - return GENERATED_FILE_HEADER.format( - source_script=source_script, - source_files=source_files, - ) - - -def relpath(a, b): - return os.path.relpath(a, b).replace('\\', '/') - - -if __name__ == '__main__': - main() diff --git a/build/reclient_cfgs/python/rewrapper_base.cfg b/build/reclient_cfgs/python/rewrapper_base.cfg deleted file mode 100644 index 556a1bb186c..00000000000 --- a/build/reclient_cfgs/python/rewrapper_base.cfg +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config. See -# //brave/build/reclient_cfgs/README.md. - -platform=OSFamily=linux,label:action_default= -canonicalize_working_dir=false -remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper -inputs=src/buildtools/reclient_cfgs/python/python_remote_wrapper,src/brave/script/import_inline.py,src/brave/script/override_utils.py diff --git a/build/reclient_cfgs/python/rewrapper_linux.cfg b/build/reclient_cfgs/python/rewrapper_linux.cfg deleted file mode 100644 index c1eb42123a4..00000000000 --- a/build/reclient_cfgs/python/rewrapper_linux.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config. See -# //brave/build/reclient_cfgs/README.md. diff --git a/build/reclient_cfgs/python/rewrapper_mac.cfg b/build/reclient_cfgs/python/rewrapper_mac.cfg deleted file mode 100644 index c1eb42123a4..00000000000 --- a/build/reclient_cfgs/python/rewrapper_mac.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config. See -# //brave/build/reclient_cfgs/README.md. diff --git a/build/reclient_cfgs/python/rewrapper_windows.cfg b/build/reclient_cfgs/python/rewrapper_windows.cfg deleted file mode 100644 index c1eb42123a4..00000000000 --- a/build/reclient_cfgs/python/rewrapper_windows.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config. See -# //brave/build/reclient_cfgs/README.md. diff --git a/build/reclient_cfgs/reproxy.cfg b/build/reclient_cfgs/reproxy.cfg deleted file mode 100644 index 6465599ec6e..00000000000 --- a/build/reclient_cfgs/reproxy.cfg +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023 The Brave Authors. All rights reserved. -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -# This config is merged with Chromium config template. See -# //brave/build/reclient_cfgs/README.md. - -# Unset Chromium variables. -service= -automatic_auth= - -# Increase verbosity for rbe debugging (temporary). -v=2 diff --git a/script/brave_license_helper.py b/script/brave_license_helper.py index 9eb7108b23c..9d1fd0ad77f 100644 --- a/script/brave_license_helper.py +++ b/script/brave_license_helper.py @@ -103,6 +103,12 @@ def AddBraveCredits(root, prune_paths, special_cases, prune_dirs, "License": "MIT", "License File": ["/brave/third_party/rapidjson/src/license.txt"], }, + os.path.join('brave', 'third_party', 'reclient_configs'): { + "Name": "reclient-configs", + "URL": "https://github.com/EngFlow/reclient-configs", + "License": "Apache-2.0", + "License File": ["/brave/third_party/reclient_configs/src/LICENSE"], + }, os.path.join('brave', 'third_party', 'rust'): { "Name": "rust-cxx", "URL": "https://crates.io/crates/cxx", diff --git a/third_party/reclient_configs/brave_custom/brave_custom.py b/third_party/reclient_configs/brave_custom/brave_custom.py new file mode 100644 index 00000000000..6152bec70da --- /dev/null +++ b/third_party/reclient_configs/brave_custom/brave_custom.py @@ -0,0 +1,112 @@ +# Copyright (c) 2023 The Brave Authors. All rights reserved. +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. + +# This file customizes reclient configs generator to support Brave builds. + +import os + +# Injected by reclient configurator. +Paths: object +ReclientCfg: object +FileUtils: object +ShellTemplate: object + + +def pre_configure(): + FileUtils.GENERATED_FILE_HEADER = FileUtils.GENERATED_FILE_HEADER.replace( + "rerun configurator", "run 'npm run sync'") + + +def merge_reproxy_cfg(reproxy_cfg): + reproxy_cfg = ReclientCfg.merge_cfg( + reproxy_cfg, + { + # Increase verbosity for rbe debugging. + 'v': 2, + }) + + # Set values from supported RBE_ environment variables. + SUPPORTED_REPROXY_ENV_VARS = ( + 'RBE_service', + 'RBE_service_no_auth', + 'RBE_tls_client_auth_cert', + 'RBE_tls_client_auth_key', + 'RBE_use_application_default_credentials', + ) + for env_var in SUPPORTED_REPROXY_ENV_VARS: + value = os.environ.get(env_var) + if value: + reproxy_cfg[env_var[4:]] = value + + return reproxy_cfg + + +def merge_rewrapper_cfg(rewrapper_cfg, tool, _host_os): + # Enabled canonicalize_working_dir mode replaces directory structure with + # `set_by_reclient/a/a` instead of `src/out/Default`. Brave builds require + # `src` dir to be named `src`, otherwise C++ overrides won't work. + rewrapper_cfg = ReclientCfg.merge_cfg(rewrapper_cfg, { + 'canonicalize_working_dir': 'false', + }) + + if tool == 'python': + # Python actions require PYTHONPATH to be set during Brave builds. We + # modify python rewrapper config to add remote wrapper into execution. + # Remote wrapper will always set PYTHONPATH before running a python + # executable. + # + # The remote wrapper is generated in generate_python_remote_wrapper() + # function below. + rewrapper_cfg = ReclientCfg.merge_cfg( + rewrapper_cfg, { + 'inputs': [ + ('{src_dir}/buildtools/reclient_cfgs/python/' + 'python_remote_wrapper'), + '{src_dir}/brave/script/import_inline.py', + '{src_dir}/brave/script/override_utils.py', + ], + 'remote_wrapper': ('{src_dir}/buildtools/reclient_cfgs/' + 'python/python_remote_wrapper'), + }) + + return rewrapper_cfg + + +def post_configure(): + generate_python_remote_wrapper() + + +# Python remote wrapper sets PYTHONPATH during remote execution. +def generate_python_remote_wrapper(): + # Load python remote wrapper template. + source_file = (f'{Paths.abspath(os.path.dirname(__file__))}/' + 'python_remote_wrapper.template') + python_remote_wrapper_template = FileUtils.read_text_file(source_file) + + # Generate PYTHONPATH env variable with paths accessible from exec_root. + relative_python_paths = [] + for python_path in os.environ.get('PYTHONPATH', '').split(os.pathsep): + if not python_path: + continue + abs_python_path = Paths.abspath(python_path) + if abs_python_path.startswith(Paths.exec_root): + relative_python_paths.append( + Paths.relpath(abs_python_path, Paths.build_dir)) + + if not relative_python_paths: + print('WARNING: PYTHONPATH is empty. Remote python actions will fail.') + + # Variables to replace in the template. + template_vars = { + 'autogenerated_header': FileUtils.create_generated_header( + (source_file, Paths.abspath(__file__))), + 'pythonpath': ':'.join(relative_python_paths), + } + + # Write the python remote wrapper. + FileUtils.write_text_file( + (f'{Paths.src_dir}/buildtools/reclient_cfgs/python/' + 'python_remote_wrapper'), + ShellTemplate(python_remote_wrapper_template).substitute(template_vars)) diff --git a/build/reclient_cfgs/python/python_remote_wrapper.template b/third_party/reclient_configs/brave_custom/python_remote_wrapper.template similarity index 90% rename from build/reclient_cfgs/python/python_remote_wrapper.template rename to third_party/reclient_configs/brave_custom/python_remote_wrapper.template index e5b0188c25d..a9202e69f84 100755 --- a/build/reclient_cfgs/python/python_remote_wrapper.template +++ b/third_party/reclient_configs/brave_custom/python_remote_wrapper.template @@ -4,7 +4,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at https://mozilla.org/MPL/2.0/. -%{AUTOGENERATED_HEADER} +%{autogenerated_header} + # WARNING: This file is a part of reclient action inputs. Any modification will # invalidate remote cache. @@ -15,7 +16,7 @@ PATH="${PATH//\\//}" export PATH="${PATH//;/:}" # Add PYTHONPATH to environment. -export PYTHONPATH="%{PYTHONPATH}" +export PYTHONPATH="%{pythonpath}" # Launch the passed command as is. "${@}"