Replace reclient configurator with EngFlow-supported one.
This commit is contained in:
Aleksey Khoroshilov
@@ -25,6 +25,7 @@ patches/**/*.patchinfo
|
||||
/third_party/bitcoin-core/src
|
||||
/third_party/playlist_component/src
|
||||
/third_party/rapidjson/src
|
||||
/third_party/reclient_configs/src
|
||||
/third_party/rust/challenge_bypass_ristretto/v1/crate
|
||||
/third_party/rust/kuchiki/v0_8/crate
|
||||
/third_party/rust/futures_retry/v0_5/crate
|
||||
|
||||
@@ -24,6 +24,7 @@ deps = {
|
||||
"third_party/bitcoin-core/src": "https://github.com/bitcoin/bitcoin.git@8105bce5b384c72cf08b25b7c5343622754e7337", # v25.0
|
||||
"third_party/argon2/src": "https://github.com/P-H-C/phc-winner-argon2.git@62358ba2123abd17fccf2a108a301d4b52c01a7c",
|
||||
"third_party/rapidjson/src": "https://github.com/Tencent/rapidjson.git@06d58b9e848c650114556a23294d0b6440078c61",
|
||||
"third_party/reclient_configs/src": "https://github.com/EngFlow/reclient-configs.git@61ddd9705962a6c6c921b42c933451d4810aa28e",
|
||||
'third_party/android_deps/libs/com_google_android_play_core': {
|
||||
'packages': [
|
||||
{
|
||||
@@ -65,7 +66,10 @@ hooks = [
|
||||
{
|
||||
'name': 'configure_reclient',
|
||||
'pattern': '.',
|
||||
'action': ['python3', 'build/reclient_cfgs/configure_reclient.py'],
|
||||
'action': ['python3', 'third_party/reclient_configs/src/configure_reclient.py',
|
||||
'--src_dir=..',
|
||||
'--exec_root=../..',
|
||||
'--custom_py=third_party/reclient_configs/brave_custom/brave_custom.py'],
|
||||
},
|
||||
{
|
||||
'name': 'download_sparkle',
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
This directory contains configuration script and reclient configs to support
|
||||
compilation on linux remotes.
|
||||
|
||||
### Configuration steps of `configure_reclient.py`
|
||||
|
||||
1. Generate reproxy and rewrapper configs.
|
||||
2. Generate `python_remote_wrapper`.
|
||||
3. If a host machine is not linux, download linux toolchain and generate
|
||||
`clang_remote_wrapper`.
|
||||
|
||||
##### Reproxy config merge
|
||||
|
||||
Reproxy config is based on the Chromium config template.
|
||||
1. Load and substitute
|
||||
`//buildtools/reclient_cfgs/reproxy_cfg_templates/reproxy.cfg.template` with
|
||||
empty variables. This is done to detect new template variables in the config.
|
||||
2. Merge `//brave/build/reclient_cfgs/reproxy.cfg`.
|
||||
3. Set auth-specific vars from `RBE_*` environment variables if they are set.
|
||||
|
||||
##### Rewrapper configs merge
|
||||
|
||||
Rewrapper configs are based on the Chromium linux config.
|
||||
1. Load `//buildtools/reclient_cfgs/linux/<tool>/rewrapper_linux.cfg`
|
||||
2. Merge `//brave/build/reclient_cfgs/<tool>/rewrapper_base.cfg`
|
||||
3. Merge `//brave/build/reclient_cfgs/<tool>/rewrapper_<host>.cfg`
|
||||
|
||||
##### Remote wrappers
|
||||
|
||||
`python_remote_wrapper` adds `PYTHONPATH` into envrionment and runs the passed
|
||||
command as is.
|
||||
|
||||
`clang_remote_wrapper` is required to run cross-compilation. It replaces default
|
||||
clang path with a linux clang path and runs the linux version of the clang
|
||||
remotely.
|
||||
|
||||
### Config merge process
|
||||
|
||||
1. Parse a config item and merge it with the existing one or create a new one.
|
||||
2. If the value is a map, overwrite map items; if it's a list, append
|
||||
list items. Map-like and list-like items are hardcoded in
|
||||
`from_reclient_cfg_value()`.
|
||||
3. If the value is empty, clear the item.
|
||||
|
||||
This allows config merger to perform such modifications:
|
||||
|
||||
```
|
||||
# Add/modify/remove items into a map-like value.
|
||||
# This adds/modifies "OSFamily" and removes "label:action_default".
|
||||
platform=OSFamily=linux,label:action_default=
|
||||
|
||||
# Add items into a list-like value.
|
||||
inputs=src/new_file,src/new_file2
|
||||
|
||||
# Modify a simple value.
|
||||
canonicalize_working_dir=false
|
||||
|
||||
# Fully replace list/map values by clearing it first.
|
||||
labels=
|
||||
labels=type=compile,compiler=clang,lang=cpp
|
||||
platform=
|
||||
platform=container-image=docker://...
|
||||
inputs=
|
||||
inputs=src/a_single_input
|
||||
```
|
||||
@@ -1,42 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
%{AUTOGENERATED_HEADER}
|
||||
# WARNING: This file is a part of reclient action inputs. Any modification will
|
||||
# invalidate remote cache.
|
||||
|
||||
set -e
|
||||
|
||||
CLANG_DIR="%{CLANG_DIR}"
|
||||
LINUX_CLANG_DIR="%{LINUX_CLANG_DIR}"
|
||||
CLANG_INCLUDE_DIR="%{CLANG_INCLUDE_DIR}"
|
||||
LINUX_CLANG_INCLUDE_DIR="%{LINUX_CLANG_INCLUDE_DIR}"
|
||||
|
||||
# Fix PATH variable when it's passed from a Windows host.
|
||||
PATH="${PATH//\\//}"
|
||||
export PATH="${PATH//;/:}"
|
||||
|
||||
# Symlink clang binary to clang++ and clang-cl.
|
||||
for clang_symlink in clang++ clang-cl; do
|
||||
if [[ ! -f $LINUX_CLANG_DIR/bin/$clang_symlink ]]; then
|
||||
ln -sr "$LINUX_CLANG_DIR/bin/clang" "$LINUX_CLANG_DIR/bin/$clang_symlink"
|
||||
fi
|
||||
done
|
||||
|
||||
# Symlink "include" directory from CLANG_INCLUDE_DIR to LINUX_CLANG_INCLUDE_DIR.
|
||||
if [[ -d $CLANG_INCLUDE_DIR && ! -d $LINUX_CLANG_INCLUDE_DIR ]]; then
|
||||
mkdir -p "$(dirname "${LINUX_CLANG_INCLUDE_DIR}")"
|
||||
ln -sr "$CLANG_INCLUDE_DIR" "$LINUX_CLANG_INCLUDE_DIR"
|
||||
fi
|
||||
|
||||
# Convert clang binary path to the linux version.
|
||||
linux_clang_binary="$1"
|
||||
linux_clang_binary="${linux_clang_binary//\\//}"
|
||||
linux_clang_binary="${linux_clang_binary/"$CLANG_DIR"/"$LINUX_CLANG_DIR"}"
|
||||
linux_clang_binary="${linux_clang_binary/.exe/}"
|
||||
|
||||
# Launch linux clang binary.
|
||||
"$linux_clang_binary" "${@:2}"
|
||||
@@ -1,10 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
|
||||
platform=OSFamily=linux,label:action_default=
|
||||
canonicalize_working_dir=false
|
||||
@@ -1,7 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
@@ -1,11 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
|
||||
remote_wrapper=../../buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper
|
||||
inputs=src/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper
|
||||
toolchain_inputs=src/third_party/llvm-build/Release+Asserts_linux/bin/clang
|
||||
@@ -1,11 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
|
||||
remote_wrapper=../../buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper
|
||||
inputs=src/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper
|
||||
toolchain_inputs=src/third_party/llvm-build/Release+Asserts_linux/bin/clang
|
||||
@@ -1,356 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(__file__).replace('\\', '/')
|
||||
SRC_DIR = os.path.abspath(f'{SCRIPT_DIR}/../../..').replace('\\', '/')
|
||||
EXEC_ROOT = os.path.abspath(f'{SRC_DIR}/..').replace('\\', '/')
|
||||
|
||||
BUILD_DIR = f'{SRC_DIR}/out/a' # Used to generate relative paths.
|
||||
RECLIENT_CFGS_DIR = f'{SRC_DIR}/buildtools/reclient_cfgs'
|
||||
CLANG_DIR = f'{SRC_DIR}/third_party/llvm-build/Release+Asserts'
|
||||
LINUX_CLANG_DIR = f'{SRC_DIR}/third_party/llvm-build/Release+Asserts_linux'
|
||||
|
||||
GENERATED_FILE_HEADER = '''# AUTOGENERATED FILE - DO NOT EDIT
|
||||
# Generated by:
|
||||
{source_script}
|
||||
#
|
||||
# To edit update:
|
||||
{source_files}
|
||||
# And run 'npm run sync'.
|
||||
'''
|
||||
|
||||
|
||||
class ShellTemplate(string.Template):
|
||||
delimiter = '%'
|
||||
|
||||
|
||||
def main():
|
||||
if not os.environ.get('RBE_service'):
|
||||
# Do nothing if RBE environment is not configured.
|
||||
return
|
||||
|
||||
# Reproxy config includes auth, and network-related parameters.
|
||||
generate_reproxy_cfg()
|
||||
# Rewrapper configs describe how different tools should be run remotely.
|
||||
generate_rewrapper_cfgs()
|
||||
# Python remote wrapper sets environment variables during remote execution.
|
||||
generate_python_remote_wrapper()
|
||||
|
||||
if not sys.platform.startswith('linux'):
|
||||
# Linux clang toolchain and clang remote wrapper are required on
|
||||
# non-linux hosts to perform cross-compilation remotely.
|
||||
download_linux_clang_toolchain()
|
||||
generate_clang_remote_wrapper()
|
||||
|
||||
|
||||
def generate_reproxy_cfg():
|
||||
# Set Chromium config template variables to empty values.
|
||||
CHROMIUM_TEMPLATE_MAPPINGS = {
|
||||
'auth_flags': '',
|
||||
'depsscanner_address': '',
|
||||
'rbe_instance': '',
|
||||
'rbe_project': '',
|
||||
}
|
||||
# Load Chromium config template.
|
||||
reproxy_cfg = parse_reclient_cfg(
|
||||
string.Template(
|
||||
read_text_file(
|
||||
f'{RECLIENT_CFGS_DIR}/reproxy_cfg_templates/'
|
||||
'reproxy.cfg.template')).substitute(CHROMIUM_TEMPLATE_MAPPINGS))
|
||||
# Merge with our config.
|
||||
source_cfg_path = f'{SCRIPT_DIR}/reproxy.cfg'
|
||||
reproxy_cfg = merge_reclient_cfg(source_cfg_path, reproxy_cfg)
|
||||
|
||||
# Use scandeps_server.
|
||||
depsscanner_address = (f'exec://{SRC_DIR}/'
|
||||
'buildtools/reclient/scandeps_server')
|
||||
if sys.platform.startswith('win'):
|
||||
depsscanner_address += '.exe'
|
||||
reproxy_cfg['depsscanner_address'] = depsscanner_address
|
||||
|
||||
# Set values from supoprted RBE_ environment variables.
|
||||
SUPPORTED_REPROXY_ENV_VARS = (
|
||||
'RBE_service',
|
||||
'RBE_service_no_auth',
|
||||
'RBE_tls_client_auth_cert',
|
||||
'RBE_tls_client_auth_key',
|
||||
'RBE_use_application_default_credentials',
|
||||
)
|
||||
for env_var in SUPPORTED_REPROXY_ENV_VARS:
|
||||
value = os.environ.get(env_var)
|
||||
if value:
|
||||
key = env_var[4:]
|
||||
reproxy_cfg[key] = from_reclient_cfg_value(key, value)
|
||||
|
||||
# Write the final config to the expected location.
|
||||
write_reclient_cfg(f'{RECLIENT_CFGS_DIR}/reproxy.cfg', reproxy_cfg,
|
||||
source_cfg_path)
|
||||
|
||||
|
||||
def generate_rewrapper_cfgs():
|
||||
# Generate chromium-browser-clang configs.
|
||||
generate_rewrapper_cfg('chromium-browser-clang', 'rewrapper_linux.cfg')
|
||||
generate_rewrapper_cfg('chromium-browser-clang', 'rewrapper_mac.cfg')
|
||||
generate_rewrapper_cfg('chromium-browser-clang', 'rewrapper_windows.cfg')
|
||||
|
||||
# Generate python configs.
|
||||
generate_rewrapper_cfg('python', 'rewrapper_linux.cfg')
|
||||
generate_rewrapper_cfg('python', 'rewrapper_mac.cfg')
|
||||
generate_rewrapper_cfg('python', 'rewrapper_windows.cfg')
|
||||
|
||||
|
||||
def generate_rewrapper_cfg(tool_dir, cfg):
|
||||
# Load Chromium config for linux remote.
|
||||
rewrapper_cfg = read_reclient_cfg(f'{RECLIENT_CFGS_DIR}/linux/{tool_dir}/'
|
||||
'rewrapper_linux.cfg')
|
||||
# Merge with our configs (rewrapper_base and rewrapper_<host>).
|
||||
source_cfg_paths = (
|
||||
f'{SCRIPT_DIR}/{tool_dir}/rewrapper_base.cfg',
|
||||
f'{SCRIPT_DIR}/{tool_dir}/{cfg}',
|
||||
)
|
||||
for source_cfg_path in source_cfg_paths:
|
||||
rewrapper_cfg = merge_reclient_cfg(source_cfg_path, rewrapper_cfg)
|
||||
# Write the final config to the expected location.
|
||||
write_reclient_cfg(f'{RECLIENT_CFGS_DIR}/{tool_dir}/{cfg}', rewrapper_cfg,
|
||||
source_cfg_paths)
|
||||
|
||||
|
||||
def generate_python_remote_wrapper():
|
||||
# Load python remote wrapper template.
|
||||
source_file = f'{SCRIPT_DIR}/python/python_remote_wrapper.template'
|
||||
python_remote_wrapper_template = read_text_file(source_file)
|
||||
|
||||
# Generate PYTHONPATH env variable with paths accessible from EXEC_ROOT.
|
||||
relative_python_paths = []
|
||||
for python_path in os.environ.get('PYTHONPATH', '').split(os.pathsep):
|
||||
if not python_path:
|
||||
continue
|
||||
abs_python_path = os.path.abspath(python_path).replace('\\', '/')
|
||||
if abs_python_path.startswith(EXEC_ROOT):
|
||||
relative_python_paths.append(relpath(abs_python_path, BUILD_DIR))
|
||||
|
||||
# Variables to replace in the template.
|
||||
template_vars = {
|
||||
'AUTOGENERATED_HEADER': create_generated_header(source_file),
|
||||
'PYTHONPATH': ':'.join(relative_python_paths),
|
||||
}
|
||||
|
||||
if not template_vars['PYTHONPATH']:
|
||||
print('WARNING: PYTHONPATH is empty. Remote python actions will fail.')
|
||||
|
||||
# Write the python remote wrapper.
|
||||
write_text_file(
|
||||
f'{SRC_DIR}/buildtools/reclient_cfgs/python/python_remote_wrapper',
|
||||
ShellTemplate(python_remote_wrapper_template).substitute(template_vars))
|
||||
|
||||
|
||||
def download_linux_clang_toolchain():
|
||||
subprocess.check_call([
|
||||
'python3',
|
||||
f'{SRC_DIR}/tools/clang/scripts/update.py',
|
||||
'--output-dir',
|
||||
f'{LINUX_CLANG_DIR}',
|
||||
'--host-os',
|
||||
'linux',
|
||||
])
|
||||
|
||||
|
||||
def generate_clang_remote_wrapper():
|
||||
if not os.path.exists(CLANG_DIR):
|
||||
raise RuntimeError(f'Cannot find clang directory at {CLANG_DIR}.')
|
||||
|
||||
# Load clang remote wrapper template.
|
||||
source_file = (f'{SCRIPT_DIR}/chromium-browser-clang/'
|
||||
'clang_remote_wrapper.template')
|
||||
clang_remote_wrapper_template = read_text_file(source_file)
|
||||
|
||||
# Find "include" directory inside clang installation. This directory will be
|
||||
# symlinked by remote wrapper for cross-compilation to work. The path is
|
||||
# clang-version dependent, so don't hardcode it.
|
||||
clang_include_dir_glob = glob.glob(f'{CLANG_DIR}/lib/**/include',
|
||||
recursive=True)
|
||||
if not clang_include_dir_glob:
|
||||
raise RuntimeError(
|
||||
f'Cannot find lib/**/include dir in {CLANG_DIR}. If clang '
|
||||
f'structure has changed, please update {__file__}.')
|
||||
clang_include_dir_abs = clang_include_dir_glob[0].replace('\\', '/')
|
||||
assert os.path.isdir(clang_include_dir_abs), clang_include_dir_abs
|
||||
clang_include_dir = relpath(clang_include_dir_abs, BUILD_DIR)
|
||||
linux_clang_include_dir = relpath(
|
||||
clang_include_dir_abs.replace(CLANG_DIR, LINUX_CLANG_DIR), BUILD_DIR)
|
||||
|
||||
# Variables to replace in the template.
|
||||
template_vars = {
|
||||
'AUTOGENERATED_HEADER': create_generated_header(source_file),
|
||||
'CLANG_DIR': relpath(CLANG_DIR, BUILD_DIR),
|
||||
'CLANG_INCLUDE_DIR': clang_include_dir,
|
||||
'LINUX_CLANG_DIR': relpath(LINUX_CLANG_DIR, BUILD_DIR),
|
||||
'LINUX_CLANG_INCLUDE_DIR': linux_clang_include_dir,
|
||||
}
|
||||
|
||||
# Write the clang remote wrapper.
|
||||
write_text_file(
|
||||
(f'{SRC_DIR}/buildtools/reclient_cfgs/chromium-browser-clang/'
|
||||
'clang_remote_wrapper'),
|
||||
ShellTemplate(clang_remote_wrapper_template).substitute(template_vars))
|
||||
|
||||
|
||||
### Reclient config manipulation helpers. ###
|
||||
def read_reclient_cfg(cfg_path):
|
||||
return dict(read_reclient_cfg_items(cfg_path))
|
||||
|
||||
|
||||
def read_reclient_cfg_items(cfg_path):
|
||||
with open(cfg_path, 'r') as f:
|
||||
yield from parse_reclient_cfg_items(f)
|
||||
|
||||
|
||||
def parse_reclient_cfg(cfg_string):
|
||||
return dict(parse_reclient_cfg_items(cfg_string.split('\n')))
|
||||
|
||||
|
||||
def parse_reclient_cfg_items(lines):
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not re.match(r'^\w+=', line):
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
yield key, from_reclient_cfg_value(key, value)
|
||||
|
||||
|
||||
def write_reclient_cfg(cfg_path, cfg, source_cfg_paths):
|
||||
cfg_to_write = create_generated_header(source_cfg_paths) + '\n'
|
||||
for key, value in cfg.items():
|
||||
formatted_value = to_reclient_cfg_value(key, value)
|
||||
if formatted_value:
|
||||
cfg_to_write += f'{formatted_value}\n'
|
||||
|
||||
write_text_file(cfg_path, cfg_to_write)
|
||||
|
||||
|
||||
def from_reclient_cfg_value(key, value):
|
||||
KEY_VALUE_PARAMS = (
|
||||
'labels',
|
||||
'platform',
|
||||
)
|
||||
LIST_PARAMS = (
|
||||
'env_var_allowlist',
|
||||
'input_list_paths',
|
||||
'inputs',
|
||||
'output_files',
|
||||
'output_directories',
|
||||
'toolchain_inputs',
|
||||
)
|
||||
|
||||
if key in KEY_VALUE_PARAMS:
|
||||
ret_val = {}
|
||||
for sub_kv in value.split(','):
|
||||
if not sub_kv:
|
||||
continue
|
||||
if '=' not in sub_kv:
|
||||
raise RuntimeError(f'key=value expected for key: {key}')
|
||||
sub_key, sub_value = sub_kv.split('=', 1)
|
||||
ret_val[sub_key] = sub_value
|
||||
return ret_val
|
||||
|
||||
if key in LIST_PARAMS:
|
||||
if not value:
|
||||
return []
|
||||
return value.split(',')
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def to_reclient_cfg_value(key, value):
|
||||
if isinstance(value, dict):
|
||||
sub_keys_values = []
|
||||
for sub_key, sub_value in value.items():
|
||||
sub_keys_values.append(to_reclient_cfg_value(sub_key, sub_value))
|
||||
return to_reclient_cfg_value(key, sub_keys_values)
|
||||
|
||||
if isinstance(value, list):
|
||||
return to_reclient_cfg_value(key, ','.join(filter(lambda v: v, value)))
|
||||
|
||||
return f'{key}={value}' if value else None
|
||||
|
||||
|
||||
def merge_reclient_cfg(cfg_path, reclient_cfg):
|
||||
for key, value in read_reclient_cfg_items(cfg_path):
|
||||
reclient_cfg = merge_reclient_cfg_item(reclient_cfg, {key: value})
|
||||
return reclient_cfg
|
||||
|
||||
|
||||
def merge_reclient_cfg_item(a, b):
|
||||
if isinstance(a, dict):
|
||||
assert isinstance(b, dict)
|
||||
if not b:
|
||||
a.clear()
|
||||
else:
|
||||
for key in b:
|
||||
if key in a:
|
||||
a[key] = merge_reclient_cfg_item(a[key], b[key])
|
||||
else:
|
||||
a[key] = b[key]
|
||||
elif isinstance(a, list):
|
||||
assert isinstance(b, list)
|
||||
if not b:
|
||||
a.clear()
|
||||
else:
|
||||
a.extend(b)
|
||||
else:
|
||||
a = b
|
||||
|
||||
return a
|
||||
|
||||
|
||||
### File helpers. ###
|
||||
def read_text_file(filepath):
|
||||
with open(filepath, 'r') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def write_text_file(filepath, data_to_write):
|
||||
if os.path.isfile(filepath):
|
||||
with open(filepath, 'r') as f:
|
||||
if f.read() == data_to_write:
|
||||
return
|
||||
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
|
||||
filepath_new = filepath + '.new'
|
||||
with open(filepath_new, 'w', newline='\n') as f:
|
||||
f.write(data_to_write)
|
||||
|
||||
shutil.move(filepath_new, filepath)
|
||||
|
||||
|
||||
def create_generated_header(source_files):
|
||||
if not isinstance(source_files, (list, tuple)):
|
||||
source_files = (source_files, )
|
||||
source_script = f'# //{relpath(__file__, SRC_DIR)}'
|
||||
source_files = '\n'.join(
|
||||
[f'# //{relpath(f, SRC_DIR)}' for f in source_files])
|
||||
|
||||
return GENERATED_FILE_HEADER.format(
|
||||
source_script=source_script,
|
||||
source_files=source_files,
|
||||
)
|
||||
|
||||
|
||||
def relpath(a, b):
|
||||
return os.path.relpath(a, b).replace('\\', '/')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,12 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
|
||||
platform=OSFamily=linux,label:action_default=
|
||||
canonicalize_working_dir=false
|
||||
remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper
|
||||
inputs=src/buildtools/reclient_cfgs/python/python_remote_wrapper,src/brave/script/import_inline.py,src/brave/script/override_utils.py
|
||||
@@ -1,7 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
@@ -1,7 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
@@ -1,7 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
@@ -1,14 +0,0 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This config is merged with Chromium config template. See
|
||||
# //brave/build/reclient_cfgs/README.md.
|
||||
|
||||
# Unset Chromium variables.
|
||||
service=
|
||||
automatic_auth=
|
||||
|
||||
# Increase verbosity for rbe debugging (temporary).
|
||||
v=2
|
||||
@@ -103,6 +103,12 @@ def AddBraveCredits(root, prune_paths, special_cases, prune_dirs,
|
||||
"License": "MIT",
|
||||
"License File": ["/brave/third_party/rapidjson/src/license.txt"],
|
||||
},
|
||||
os.path.join('brave', 'third_party', 'reclient_configs'): {
|
||||
"Name": "reclient-configs",
|
||||
"URL": "https://github.com/EngFlow/reclient-configs",
|
||||
"License": "Apache-2.0",
|
||||
"License File": ["/brave/third_party/reclient_configs/src/LICENSE"],
|
||||
},
|
||||
os.path.join('brave', 'third_party', 'rust'): {
|
||||
"Name": "rust-cxx",
|
||||
"URL": "https://crates.io/crates/cxx",
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# Copyright (c) 2023 The Brave Authors. All rights reserved.
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This file customizes reclient configs generator to support Brave builds.
|
||||
|
||||
import os
|
||||
|
||||
# Injected by reclient configurator.
|
||||
Paths: object
|
||||
ReclientCfg: object
|
||||
FileUtils: object
|
||||
ShellTemplate: object
|
||||
|
||||
|
||||
def pre_configure():
|
||||
FileUtils.GENERATED_FILE_HEADER = FileUtils.GENERATED_FILE_HEADER.replace(
|
||||
"rerun configurator", "run 'npm run sync'")
|
||||
|
||||
|
||||
def merge_reproxy_cfg(reproxy_cfg):
|
||||
reproxy_cfg = ReclientCfg.merge_cfg(
|
||||
reproxy_cfg,
|
||||
{
|
||||
# Increase verbosity for rbe debugging.
|
||||
'v': 2,
|
||||
})
|
||||
|
||||
# Set values from supported RBE_ environment variables.
|
||||
SUPPORTED_REPROXY_ENV_VARS = (
|
||||
'RBE_service',
|
||||
'RBE_service_no_auth',
|
||||
'RBE_tls_client_auth_cert',
|
||||
'RBE_tls_client_auth_key',
|
||||
'RBE_use_application_default_credentials',
|
||||
)
|
||||
for env_var in SUPPORTED_REPROXY_ENV_VARS:
|
||||
value = os.environ.get(env_var)
|
||||
if value:
|
||||
reproxy_cfg[env_var[4:]] = value
|
||||
|
||||
return reproxy_cfg
|
||||
|
||||
|
||||
def merge_rewrapper_cfg(rewrapper_cfg, tool, _host_os):
|
||||
# Enabled canonicalize_working_dir mode replaces directory structure with
|
||||
# `set_by_reclient/a/a` instead of `src/out/Default`. Brave builds require
|
||||
# `src` dir to be named `src`, otherwise C++ overrides won't work.
|
||||
rewrapper_cfg = ReclientCfg.merge_cfg(rewrapper_cfg, {
|
||||
'canonicalize_working_dir': 'false',
|
||||
})
|
||||
|
||||
if tool == 'python':
|
||||
# Python actions require PYTHONPATH to be set during Brave builds. We
|
||||
# modify python rewrapper config to add remote wrapper into execution.
|
||||
# Remote wrapper will always set PYTHONPATH before running a python
|
||||
# executable.
|
||||
#
|
||||
# The remote wrapper is generated in generate_python_remote_wrapper()
|
||||
# function below.
|
||||
rewrapper_cfg = ReclientCfg.merge_cfg(
|
||||
rewrapper_cfg, {
|
||||
'inputs': [
|
||||
('{src_dir}/buildtools/reclient_cfgs/python/'
|
||||
'python_remote_wrapper'),
|
||||
'{src_dir}/brave/script/import_inline.py',
|
||||
'{src_dir}/brave/script/override_utils.py',
|
||||
],
|
||||
'remote_wrapper': ('{src_dir}/buildtools/reclient_cfgs/'
|
||||
'python/python_remote_wrapper'),
|
||||
})
|
||||
|
||||
return rewrapper_cfg
|
||||
|
||||
|
||||
def post_configure():
|
||||
generate_python_remote_wrapper()
|
||||
|
||||
|
||||
# Python remote wrapper sets PYTHONPATH during remote execution.
|
||||
def generate_python_remote_wrapper():
|
||||
# Load python remote wrapper template.
|
||||
source_file = (f'{Paths.abspath(os.path.dirname(__file__))}/'
|
||||
'python_remote_wrapper.template')
|
||||
python_remote_wrapper_template = FileUtils.read_text_file(source_file)
|
||||
|
||||
# Generate PYTHONPATH env variable with paths accessible from exec_root.
|
||||
relative_python_paths = []
|
||||
for python_path in os.environ.get('PYTHONPATH', '').split(os.pathsep):
|
||||
if not python_path:
|
||||
continue
|
||||
abs_python_path = Paths.abspath(python_path)
|
||||
if abs_python_path.startswith(Paths.exec_root):
|
||||
relative_python_paths.append(
|
||||
Paths.relpath(abs_python_path, Paths.build_dir))
|
||||
|
||||
if not relative_python_paths:
|
||||
print('WARNING: PYTHONPATH is empty. Remote python actions will fail.')
|
||||
|
||||
# Variables to replace in the template.
|
||||
template_vars = {
|
||||
'autogenerated_header': FileUtils.create_generated_header(
|
||||
(source_file, Paths.abspath(__file__))),
|
||||
'pythonpath': ':'.join(relative_python_paths),
|
||||
}
|
||||
|
||||
# Write the python remote wrapper.
|
||||
FileUtils.write_text_file(
|
||||
(f'{Paths.src_dir}/buildtools/reclient_cfgs/python/'
|
||||
'python_remote_wrapper'),
|
||||
ShellTemplate(python_remote_wrapper_template).substitute(template_vars))
|
||||
+3
-2
@@ -4,7 +4,8 @@
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
%{AUTOGENERATED_HEADER}
|
||||
%{autogenerated_header}
|
||||
|
||||
# WARNING: This file is a part of reclient action inputs. Any modification will
|
||||
# invalidate remote cache.
|
||||
|
||||
@@ -15,7 +16,7 @@ PATH="${PATH//\\//}"
|
||||
export PATH="${PATH//;/:}"
|
||||
|
||||
# Add PYTHONPATH to environment.
|
||||
export PYTHONPATH="%{PYTHONPATH}"
|
||||
export PYTHONPATH="%{pythonpath}"
|
||||
|
||||
# Launch the passed command as is.
|
||||
"${@}"
|
||||
Reference in New Issue
Block a user