From b588d35f91f00aeb7034cd6566fd582e2cce4707 Mon Sep 17 00:00:00 2001 From: bridiver Date: Fri, 18 Mar 2022 14:24:58 -0700 Subject: [PATCH] download hermetic xcode --- DEPS | 13 +-- build/mac/download_hermetic_xcode.py | 149 +++++++++++++++++++++++++++ script/deps.py | 7 ++ script/hermetic_xcode.py | 92 ----------------- 4 files changed, 163 insertions(+), 98 deletions(-) create mode 100644 build/mac/download_hermetic_xcode.py delete mode 100644 script/hermetic_xcode.py diff --git a/DEPS b/DEPS index ba661f4b55a..9c03393cd66 100644 --- a/DEPS +++ b/DEPS @@ -32,6 +32,13 @@ hooks = [ 'pattern': '.', 'action': ['python', 'script/bootstrap.py'], }, + { + # Download hermetic xcode for goma + 'name': 'download_hermetic_xcode', + 'pattern': '.', + 'condition': 'host_os == "mac"', + 'action': ['vpython3', 'build/mac/download_hermetic_xcode.py'], + }, { # Download rust deps if necessary for Android 'name': 'download_rust_deps', @@ -62,12 +69,6 @@ hooks = [ 'condition': 'not checkout_android and not checkout_ios', 'action': ['vpython3', 'script/web_discovery_project.py', '--install'], }, - { - 'name': 'hermetic_xcode', - 'pattern': '.', - 'condition': 'checkout_mac', - 'action': ['vpython3', 'script/hermetic_xcode.py'], - }, { 'name': 'generate_licenses', 'pattern': '.', diff --git a/build/mac/download_hermetic_xcode.py b/build/mac/download_hermetic_xcode.py new file mode 100644 index 00000000000..5fda5a63376 --- /dev/null +++ b/build/mac/download_hermetic_xcode.py @@ -0,0 +1,149 @@ +#!/usr/bin/env vpython3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import pkg_resources +import platform +import plistlib +import subprocess +import sys + +try: + from urllib2 import URLError +except ImportError: # For Py3 compatibility + from urllib.error import URLError # pylint: disable=no-name-in-module,import-error + +import deps +from deps_config import DEPS_PACKAGES_URL + +def LoadPList(path): + """Loads Plist at |path| and returns it as a dictionary.""" + if sys.version_info.major == 2: + return plistlib.readPlist(path) + with open(path, 'rb') as f: + return plistlib.load(f) + +# This contains binaries from Xcode 13.2.1 13C100, along with the macOS 12 SDK +XCODE_VERSION = '13.2.1' +HERMETIC_XCODE_BINARY = DEPS_PACKAGES_URL + '/xcode-hermetic-toolchain/xcode-hermetic-toolchain-xcode-' + XCODE_VERSION + '-sdk-12.1-12.0.tar.gz' + +# The toolchain will not be downloaded if the minimum OS version is not met. 19 +# is the major version number for macOS 10.15. Xcode 13.2 13C90 only runs on +# 11.3 and newer, but some bots are still running older OS versions. 10.15.4, +# the OS minimum through Xcode 12.4, still seems to work. +MAC_MINIMUM_OS_VERSION = [19, 4] + +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'build')) +TOOLCHAIN_ROOT = os.path.join(BASE_DIR, 'mac_files') +TOOLCHAIN_BUILD_DIR = os.path.join(TOOLCHAIN_ROOT, 'Xcode.app') + + +def PlatformMeetsHermeticXcodeRequirements(): + if sys.platform != 'darwin': + return True + needed = MAC_MINIMUM_OS_VERSION + major_version = [int(v) for v in platform.release().split('.')[:len(needed)]] + return major_version >= needed + + +def GetHermeticXcodeVersion(binaries_root): + hermetic_xcode_version_plist_path = os.path.join(binaries_root, + 'Contents/version.plist') + + if not os.path.exists(hermetic_xcode_version_plist_path): + return '' + + hermetic_xcode_version_plist = LoadPList(hermetic_xcode_version_plist_path) + return hermetic_xcode_version_plist['CFBundleShortVersionString'] + + +def InstallXcodeBinaries(): + """Installs the Xcode binaries and accepts the license. + """ + binaries_root = os.path.join(TOOLCHAIN_ROOT, 'xcode_binaries') + if (XCODE_VERSION == GetHermeticXcodeVersion(binaries_root) and not + os.path.islink(binaries_root)): + print('Hermetic Xcode ' + XCODE_VERSION + ' already installed') + return 0 + + url = HERMETIC_XCODE_BINARY + print('Downloading hermetic xcode: %s' % url) + try: + deps.DownloadAndUnpack(url, binaries_root) + except URLError: + print('Failed to download hermetic Xcode: %s' % url) + print('Exiting.') + return 1 + + # Accept the license for this version of Xcode if it's newer than the + # currently accepted version. + hermetic_xcode_version = GetHermeticXcodeVersion(binaries_root) + + hermetic_xcode_license_path = os.path.join(binaries_root, + 'Contents/Resources/LicenseInfo.plist') + hermetic_xcode_license_plist = LoadPList(hermetic_xcode_license_path) + hermetic_xcode_license_version = hermetic_xcode_license_plist['licenseID'] + + should_overwrite_license = True + current_license_path = '/Library/Preferences/com.apple.dt.Xcode.plist' + if os.path.exists(current_license_path): + current_license_plist = LoadPList(current_license_path) + xcode_version = current_license_plist.get( + 'IDEXcodeVersionForAgreedToGMLicense') + if (xcode_version is not None and pkg_resources.parse_version(xcode_version) + >= pkg_resources.parse_version(hermetic_xcode_version)): + should_overwrite_license = False + + if not should_overwrite_license: + return 0 + + # Use puppet's sudoers script to accept the license if its available. + license_accept_script = '/usr/local/bin/xcode_accept_license.py' + if os.path.exists(license_accept_script): + args = [ + 'sudo', license_accept_script, '--xcode-version', hermetic_xcode_version, + '--license-version', hermetic_xcode_license_version + ] + subprocess.check_call(args) + return 0 + + # Otherwise manually accept the license. This will prompt for sudo. + print('Accepting new Xcode license. Requires sudo.') + sys.stdout.flush() + args = [ + 'sudo', 'defaults', 'write', current_license_path, + 'IDEXcodeVersionForAgreedToGMLicense', hermetic_xcode_version + ] + subprocess.check_call(args) + args = [ + 'sudo', 'defaults', 'write', current_license_path, + 'IDELastGMLicenseAgreedTo', hermetic_xcode_license_version + ] + subprocess.check_call(args) + args = ['sudo', 'plutil', '-convert', 'xml1', current_license_path] + subprocess.check_call(args) + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description='Download hermetic Xcode.') + args = parser.parse_args() + + if not PlatformMeetsHermeticXcodeRequirements(): + print('OS version does not support toolchain.') + return 0 + + return InstallXcodeBinaries() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/script/deps.py b/script/deps.py index 19fb65efb34..8ffb016f683 100755 --- a/script/deps.py +++ b/script/deps.py @@ -6,6 +6,7 @@ """This script is used to download deps.""" import os +import shutil import sys import tarfile import tempfile @@ -71,6 +72,12 @@ def DownloadAndUnpack(url, output_dir, path_prefix=None): with tempfile.TemporaryFile() as f: DownloadUrl(url, f) f.seek(0) + # TODO(bridiver) we need to validate against a checksum + try: + os.unlink(output_dir) + except OSError as e: + pass + shutil.rmtree(output_dir, ignore_errors=True) EnsureDirExists(output_dir) if url.endswith('.zip'): assert path_prefix is None diff --git a/script/hermetic_xcode.py b/script/hermetic_xcode.py deleted file mode 100644 index 9198b3d8f48..00000000000 --- a/script/hermetic_xcode.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -"""Script to put a hermetic Xcode toolchain into the source tree.""" - -import os -import subprocess -import sys - - -BRAVE_CORE_ROOT = os.path.abspath( - os.path.join(os.path.dirname(__file__), os.pardir)) - -CHROMIUM_SRC_ROOT = os.path.abspath( - os.path.join(BRAVE_CORE_ROOT, os.pardir)) - -MAC_FILES = os.path.join(CHROMIUM_SRC_ROOT, 'build', 'mac_files') - - -def _run(*args, workdir=None, extra_env={}): - # Set environment variables for subprocess - env = os.environ.copy() - env.update(extra_env) - - try: - result = subprocess.run( - args, - cwd=workdir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True) - return result.stdout, result.stderr - except subprocess.CalledProcessError as e: - print(e.output) - raise e - - -def _normalize_xcode_version(version): - parts = version.split('.') - while len(parts) < 3: - parts.append('0') - return '.'.join(parts) - - -def _get_xcode_version(): - # stdout should be of format - # Xcode x.y.z - # Build version abc - stdout, _ = _run('xcodebuild', '-version') - - version = [] - for line in stdout.split(b'\n'): - if line.startswith(b'Xcode '): - xcode = _normalize_xcode_version( - str(line[len(b'Xcode '):], 'utf8')) - version.append(xcode) - elif line.startswith(b'Build version '): - build = str(line[len(b'Build version '):], 'utf8') - version.append(build) - - return '.'.join(version) - - -def _get_install_path(): - xcode_version = _get_xcode_version() - - # TODO(yannic): Use xcode-locator to ensure it's the right version? - stdout, _ = _run('xcode-select', '--print-path') - - developer_dir = str(stdout, 'utf8') - return { - 'version': xcode_version, - 'path': os.path.abspath( - os.path.join(developer_dir, os.pardir, os.pardir)) - } - - -def main(): - if os.path.exists(MAC_FILES): - _run('rm', '-rf', MAC_FILES) - _run('mkdir', '-p', MAC_FILES) - - xcode = _get_install_path() - _run('ln', '-s', xcode['path'], 'xcode_binaries', workdir=MAC_FILES) - - return 0 - - -if __name__ == '__main__': - sys.exit(main())