Discharge7443/brave-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add npm run build_chromium (#21017)

Browse Source 
* Add npm run build_chromium

* Per platfrom targets

* Add target_os & target_arch

* Fix android apk path

* Use repack-archive.py

* Use chromeVersion for output

* Change the output filename, add mac support

* Add comments

* Support linux

* Adjust build args

* Fix presubmit

* Fix review comment

* Add isChromium to config.js

* Remove changing outputDir

* Return <build_dir>

* Tune gn args

* Review fixes

* Fix non-mac builds

* Rename the script, add a warning

* Change the comment
This commit is contained in:
Mikhail
2023-11-28 13:11:00 -05:00
committed by GitHub
parent 292edbea5e
commit 3a923c5018
6 changed files with 403 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/commands/lib/buildChromiumRelease.js
+175
View File
@@ -0,0 +1,175 @@
// Copyright (c) 2023 The Brave Authors. All rights reserved.
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// A script to build and pack a Chromium release build from scratch
// Reuses the same /src folder
// Designed to be used on CI, but should work locally too.
// The script includes syncing; there is no need to run npm run sync before.
const config = require('./config')
const util = require('./util')
const path = require('path')
const fs = require('fs-extra')
const syncUtil = require('./syncUtils')
const Log = require('./logging')
// Use the same filename as for Brave archive.
const getOutputFilename = () => {
const platform = (() => {
if (config.getTargetOS() === 'win')
return 'win32'
if (config.getTargetOS() === 'mac')
return 'darwin'
return config.getTargetOS()
})()
return `chromium-${config.chromeVersion}-${platform}-${config.targetArch}`
}
const chromiumConfigs = {
'win': {
buildTarget: 'mini_installer',
processArtifacts: () => {
// Repack it to reduce the size and use .zip instead of .7z.
input = path.join(config.outputDir, 'chrome.7z')
output = path.join(config.outputDir, `${getOutputFilename()}.zip`)
util.run('python3',
[
path.join(config.braveCoreDir, 'script', 'repack-archive.py'),
`--input=${input}`,
`--output=${output}`,
'--target_dir=Chrome-bin',
],
config.defaultOptions)
}
},
'linux': {
buildTarget: 'chrome/installer/linux:stable_deb',
processArtifacts: () => {
const debArch = (() => {
if (config.targetArch === 'x64') return 'amd64'
return config.targetArch
})()
fs.moveSync(
path.join(config.outputDir,
`chromium-browser-stable_${config.chromeVersion}-1_${debArch}.deb`),
path.join(config.outputDir, `${getOutputFilename()}.deb`))
}
},
'mac': {
buildTarget: 'chrome',
extraHooks: () => {
Log.progressScope('download_hermetic_xcode', () => {
util.run('vpython3',
[
path.join(config.braveCoreDir,
'build', 'mac', 'download_hermetic_xcode.py'),
],
config.defaultOptions)
})
},
processArtifacts: () => {
util.run('zip',
['-r', '-y', `${getOutputFilename()}.zip`, 'Chromium.app'],
{ cwd: config.outputDir }
)
}
},
'android': {
buildTarget: 'chrome_public_apk',
processArtifacts: () => {
fs.moveSync(
path.join(config.outputDir, 'apks', 'ChromePublic.apk'),
path.join(config.outputDir, `${getOutputFilename()}.apk`))
}
},
}
// A function to make gn args to build a release Chromium build.
// There is two primarily sources:
// 1. Chromium perf builds: tools/mb/mb_config_expectations/chromium.perf.json
// 2. Brave Release build configuration
function getChromiumGnArgs() {
const targetOs = config.getTargetOS()
const targetArch = config.targetArch
const args = {
target_cpu: targetArch,
target_os: targetOs,
is_official_build: true,
enable_keystone_registration_framework: false,
ffmpeg_branding: 'Chrome',
enable_widevine: true,
ignore_missing_widevine_signing_cert: true,
...config.extraGnArgs,
}
if (targetOs === 'android') {
args.debuggable_apks = false
} else {
args.enable_hangout_services_extension = true
args.enable_nacl = false
}
if (targetOs === 'mac') {
args.use_system_xcode = true
}
return args
}
function buildChromiumRelease(buildOptions = {}) {
if (!config.isCI && !buildOptions.force) {
console.error(
'Warning: the command resets all changes in src/ folder.\n' +
'src/brave stays untouched. Pass --force to continue.')
return 1
}
config.buildConfig = 'Release'
config.isChromium = true
config.update(buildOptions)
const chromiumConfig = chromiumConfigs[config.getTargetOS()]
if (chromiumConfig == undefined)
throw Error(`${config.getTargetOS()} is unsupported`)
syncUtil.maybeInstallDepotTools()
syncUtil.buildDefaultGClientConfig(
[config.getTargetOS()], [config.targetArch], true)
util.runGit(config.srcDir, ['clean', '-f', '-d'])
Log.progressScope('gclient sync', () => {
syncUtil.syncChromium(true, true, false)
})
Log.progressScope('gclient runhooks', () => {
util.runGClient(['runhooks'])
})
if (chromiumConfig.extraHooks != undefined) {
chromiumConfig.extraHooks()
}
const options = config.defaultOptions
const buildArgsStr = util.buildArgsToString(getChromiumGnArgs())
util.run('gn', ['gen', config.outputDir, '--args="' + buildArgsStr + '"'],
options)
Log.progressScope(`ninja`, () => {
const target = chromiumConfig.buildTarget
const ninjaOpts = [
'-C', options.outputDir || config.outputDir, target,
...config.extraNinjaOpts
]
util.run('autoninja', ninjaOpts, config.defaultOptions)
})
Log.progressScope('make archive', () => {
chromiumConfig.processArtifacts()
})
}
module.exports = buildChromiumRelease
build/commands/lib/config.js
+4
View File
@@ -107,6 +107,7 @@ const Config = function () {
this.buildTarget = 'brave'
this.rootDir = rootDir
this.isUniversalBinary = false
this.isChromium = false
this.scriptDir = path.join(this.rootDir, 'scripts')
this.srcDir = path.join(this.rootDir, 'src')
this.chromeVersion = this.getProjectVersion('chrome')
@@ -1289,6 +1290,9 @@ Object.defineProperty(Config.prototype, 'outputDir', {
if (this.targetEnvironment) {
buildConfigDir = buildConfigDir + "_" + this.targetEnvironment
}
if (this.isChromium) {
buildConfigDir = buildConfigDir + "_chromium"
}
return path.join(baseDir, buildConfigDir)
},
build/commands/lib/syncUtils.js
+197
View File
@@ -0,0 +1,197 @@
// Copyright (c) 2023 The Brave Authors. All rights reserved.
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
const chalk = require('chalk')
const config = require('./config')
const fs = require('fs')
const path = require('path')
const Log = require('./logging')
const util = require('./util')
function maybeInstallDepotTools(options = config.defaultOptions) {
options.cwd = config.braveCoreDir
if (!fs.existsSync(config.depotToolsDir)) {
Log.progressScope('install depot_tools', () => {
fs.mkdirSync(config.depotToolsDir)
util.run(
'git',
[
'-C',
config.depotToolsDir,
'clone',
'https://chromium.googlesource.com/chromium/tools/depot_tools.git',
'.'
],
options
)
})
}
const ninjaLogCfgPath = path.join(config.depotToolsDir, 'ninjalog.cfg');
if (!fs.existsSync(ninjaLogCfgPath)) {
// Create a ninja config to prevent (auto)ninja from calling goma_auth
// each time. See for details:
// https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md
const ninjaLogCfgConfig = {
'is-googler': false,
'version': 3,
'countdown': 10,
'opt-in': false,
};
fs.writeFileSync(ninjaLogCfgPath, JSON.stringify(ninjaLogCfgConfig))
}
}
function toGClientConfigItem(name, value, pretty = true) {
// Convert value to json and replace "%True%" -> True, "%False%" -> False,
// "%None%" -> None.
const pythonLikeValue =
JSON.stringify(value, null, pretty ? 2 : 0).replace(/"%(.*?)%"/gm, '$1')
return `${name} = ${pythonLikeValue}\n`
}
function buildDefaultGClientConfig(
targetOSList, targetArchList, onlyChromium = false) {
const items = [
{
managed: '%False%',
name: 'src',
url: config.chromiumRepo,
custom_deps: {
'src/third_party/WebKit/LayoutTests': '%None%',
'src/chrome_frame/tools/test/reference_build/chrome': '%None%',
'src/chrome_frame/tools/test/reference_build/chrome_win': '%None%',
'src/chrome/tools/test/reference_build/chrome': '%None%',
'src/chrome/tools/test/reference_build/chrome_linux': '%None%',
'src/chrome/tools/test/reference_build/chrome_mac': '%None%',
'src/chrome/tools/test/reference_build/chrome_win': '%None%'
},
custom_vars: {
'checkout_rust': '%True%',
'checkout_pgo_profiles': config.isBraveReleaseBuild() ? '%True%' :
'%False%'
}
}
]
if (!onlyChromium) {
items.push({
managed: '%False%',
name: 'src/brave',
// We do not use gclient to manage brave-core, so this should not
// actually get used.
url: 'https://github.com/brave/brave-core.git'
})
}
let out = toGClientConfigItem('solutions', items);
if (process.env.GIT_CACHE_PATH) {
out += toGClientConfigItem('cache_dir', process.env.GIT_CACHE_PATH)
}
if (targetOSList) {
out += toGClientConfigItem('target_os', targetOSList, false)
}
if (targetArchList) {
out += toGClientConfigItem('target_cpu', targetArchList, false)
}
fs.writeFileSync(config.defaultGClientFile, out)
}
function shouldUpdateChromium(latestSyncInfo, expectedSyncInfo) {
const chromiumRef = expectedSyncInfo.chromiumRef
const headSHA = util.runGit(config.srcDir, ['rev-parse', 'HEAD'], true)
const targetSHA = util.runGit(config.srcDir, ['rev-parse', chromiumRef], true)
const needsUpdate = targetSHA !== headSHA || (!headSHA && !targetSHA) ||
JSON.stringify(latestSyncInfo) !== JSON.stringify(expectedSyncInfo)
if (needsUpdate) {
const currentRef = util.getGitReadableLocalRef(config.srcDir)
console.log(`Chromium repo ${chalk.blue.bold('needs sync')}.\n target is ${
chalk.italic(chromiumRef)} at commit ${
targetSHA || '[missing]'}\n current commit is ${
chalk.italic(currentRef || '[unknown]')} at commit ${
chalk.inverse(headSHA || '[missing]')}\n latest successful sync is ${
JSON.stringify(latestSyncInfo, null, 4)}`)
}
else {
console.log(
chalk.green.bold(`Chromium repo does not need sync as it is already ${
chalk.italic(chromiumRef)} at commit ${targetSHA || '[missing]'}.`))
}
return needsUpdate
}
function syncChromium(syncWithForce, sync_chromium, delete_unused_deps) {
const requiredChromiumRef = config.getProjectRef('chrome')
let args = [
'sync', '--nohooks', '--revision', 'src@' + requiredChromiumRef, '--reset',
'--with_tags', '--with_branch_heads', '--upstream'
];
if (syncWithForce) {
args.push('--force')
}
const latestSyncInfoFilePath =
path.join(config.rootDir, '.brave_latest_successful_sync.json')
const latestSyncInfo = util.readJSON(latestSyncInfoFilePath, {})
const expectedSyncInfo = {
chromiumRef: requiredChromiumRef,
gClientTimestamp: fs.statSync(config.gClientFile).mtimeMs.toString(),
}
const chromiumNeedsUpdate =
shouldUpdateChromium(latestSyncInfo, expectedSyncInfo)
const shouldSyncChromium = chromiumNeedsUpdate || syncWithForce
if (!shouldSyncChromium && !sync_chromium) {
if (delete_unused_deps && !config.isCI) {
Log.warn(
'--delete_unused_deps is ignored for src/ dir because Chromium sync ' +
'is required. Pass --sync_chromium to force it.')
}
return false
}
if (delete_unused_deps) {
if (util.isGitExclusionExists(config.srcDir, 'brave/')) {
args.push('-D')
} else if (!config.isCI) {
Log.warn(
'--delete_unused_deps is ignored because sync has not yet added ' +
'the exclusion for the src/brave/ directory, likely because sync ' +
'has not previously successfully run before.')
}
}
if (sync_chromium !== undefined) {
if (!sync_chromium) {
Log.warn(
'Chromium needed sync but received the flag to skip performing the ' +
'update. Working directory may not compile correctly.')
return false
} else if (!shouldSyncChromium) {
Log.warn(
'Chromium doesn\'t need sync but received the flag to do it anyway.')
}
}
util.runGClient(args)
util.addGitExclusion(config.srcDir, 'brave/')
util.writeJSON(latestSyncInfoFilePath, expectedSyncInfo)
const postSyncChromiumRef = util.getGitReadableLocalRef(config.srcDir)
Log.status(`Chromium is now at ${postSyncChromiumRef || '[unknown]'}`)
return true
}
module.exports = {
maybeInstallDepotTools,
buildDefaultGClientConfig,
syncChromium
}
build/commands/scripts/commands.js
+19
View File
@@ -9,6 +9,7 @@ const fs = require('fs-extra')
const config = require('../lib/config')
const util = require('../lib/util')
const build = require('../lib/build')
const buildChromiumRelease = require('../lib/buildChromiumRelease')
const { buildFuzzer, runFuzzer } = require('../lib/fuzzer')
const versions = require('../lib/versions')
const start = require('../lib/start')
@@ -166,6 +167,24 @@ program
.arguments('[build_config]')
.action(build)
program
.command('build_chromium_release')
.description(
'Produces a chromium release build for performance testing.\n' +
'Uses the same /src directory; all brave patches are reverted.\n' +
'The default build_dir is `chromium_Release(_target_arch)`.\n' +
'Intended for use on CI, use locally with care.')
.option('--force', 'Ignore a warning about non-CI build')
.option('-C <build_dir>', 'build config (out/chromium_Release')
.option('--target_os <target_os>', 'target OS')
.option('--target_arch <target_arch>', 'target architecture')
.option('--gn <arg>', 'Additional gn args, in the form <key>:<value>',
collect, [])
.option('--ninja <opt>',
'Additional Ninja command-line options, in the form <key>:<value>',
collect, [])
.action(buildChromiumRelease)
program
.command('create_dist')
.option('-C <build_dir>', 'build config (out/Debug, out/Release')
build/commands/scripts/sync.js
+7 -177
View File
@@ -9,7 +9,7 @@ const path = require('path')
const config = require('../lib/config')
const util = require('../lib/util')
const Log = require('../lib/logging')
const chalk = require('chalk')
const syncUtil = require('../lib/syncUtils')
program
.version(process.env.npm_package_version)
@@ -25,179 +25,6 @@ program
.option('-D, --delete_unused_deps', 'delete from the working copy any dependencies that have been removed since the last sync')
.option('--nohooks', 'Do not run hooks after updating')
function maybeInstallDepotTools(options = config.defaultOptions) {
options.cwd = config.braveCoreDir
if (!fs.existsSync(config.depotToolsDir)) {
Log.progressScope('install depot_tools', () => {
fs.mkdirSync(config.depotToolsDir)
util.run(
'git',
[
'-C',
config.depotToolsDir,
'clone',
'https://chromium.googlesource.com/chromium/tools/depot_tools.git',
'.'
],
options
)
})
}
const ninjaLogCfgPath = path.join(config.depotToolsDir, 'ninjalog.cfg');
if (!fs.existsSync(ninjaLogCfgPath)) {
// Create a ninja config to prevent (auto)ninja from calling goma_auth
// each time. See for details:
// https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md
const ninjaLogCfgConfig = {
'is-googler': false,
'version': 3,
'countdown': 10,
'opt-in': false,
};
fs.writeFileSync(ninjaLogCfgPath, JSON.stringify(ninjaLogCfgConfig))
}
}
function toGClientConfigItem(name, value, pretty = true) {
// Convert value to json and replace "%True%" -> True, "%False%" -> False,
// "%None%" -> None.
const pythonLikeValue =
JSON.stringify(value, null, pretty ? 2 : 0).replace(/"%(.*?)%"/gm, '$1')
return `${name} = ${pythonLikeValue}\n`
}
function buildDefaultGClientConfig(targetOSList, targetArchList) {
let out = toGClientConfigItem('solutions', [
{
managed: '%False%',
name: 'src',
url: config.chromiumRepo,
custom_deps: {
'src/third_party/WebKit/LayoutTests': '%None%',
'src/chrome_frame/tools/test/reference_build/chrome': '%None%',
'src/chrome_frame/tools/test/reference_build/chrome_win': '%None%',
'src/chrome/tools/test/reference_build/chrome': '%None%',
'src/chrome/tools/test/reference_build/chrome_linux': '%None%',
'src/chrome/tools/test/reference_build/chrome_mac': '%None%',
'src/chrome/tools/test/reference_build/chrome_win': '%None%'
},
custom_vars: {
'checkout_rust': '%True%',
'checkout_pgo_profiles': config.isBraveReleaseBuild() ? '%True%' :
'%False%'
}
},
{
managed: '%False%',
name: 'src/brave',
// We do not use gclient to manage brave-core, so this should not
// actually get used.
url: 'https://github.com/brave/brave-core.git'
}
])
if (process.env.GIT_CACHE_PATH) {
out += toGClientConfigItem('cache_dir', process.env.GIT_CACHE_PATH)
}
if (targetOSList) {
out += toGClientConfigItem('target_os', targetOSList, false)
}
if (targetArchList) {
out += toGClientConfigItem('target_cpu', targetArchList, false)
}
fs.writeFileSync(config.defaultGClientFile, out)
}
function shouldUpdateChromium(latestSyncInfo, expectedSyncInfo) {
const chromiumRef = expectedSyncInfo.chromiumRef
const headSHA = util.runGit(config.srcDir, ['rev-parse', 'HEAD'], true)
const targetSHA = util.runGit(config.srcDir, ['rev-parse', chromiumRef], true)
const needsUpdate = targetSHA !== headSHA || (!headSHA && !targetSHA) ||
JSON.stringify(latestSyncInfo) !== JSON.stringify(expectedSyncInfo)
if (needsUpdate) {
const currentRef = util.getGitReadableLocalRef(config.srcDir)
console.log(`Chromium repo ${chalk.blue.bold('needs sync')}.\n target is ${
chalk.italic(chromiumRef)} at commit ${
targetSHA || '[missing]'}\n current commit is ${
chalk.italic(currentRef || '[unknown]')} at commit ${
chalk.inverse(headSHA || '[missing]')}\n latest successful sync is ${
JSON.stringify(latestSyncInfo, null, 4)}`)
}
else {
console.log(
chalk.green.bold(`Chromium repo does not need sync as it is already ${
chalk.italic(chromiumRef)} at commit ${targetSHA || '[missing]'}.`))
}
return needsUpdate
}
function syncChromium(program) {
const requiredChromiumRef = config.getProjectRef('chrome')
let args = [
'sync', '--nohooks', '--revision', 'src@' + requiredChromiumRef, '--reset',
'--with_tags', '--with_branch_heads', '--upstream'
];
const syncWithForce = program.init || program.force
if (syncWithForce) {
args.push('--force')
}
const latestSyncInfoFilePath =
path.join(config.rootDir, '.brave_latest_successful_sync.json')
const latestSyncInfo = util.readJSON(latestSyncInfoFilePath, {})
const expectedSyncInfo = {
chromiumRef: requiredChromiumRef,
gClientTimestamp: fs.statSync(config.gClientFile).mtimeMs.toString(),
}
const chromiumNeedsUpdate =
shouldUpdateChromium(latestSyncInfo, expectedSyncInfo)
const shouldSyncChromium = chromiumNeedsUpdate || syncWithForce
if (!shouldSyncChromium && !program.sync_chromium) {
if (program.delete_unused_deps && !config.isCI) {
Log.warn(
'--delete_unused_deps is ignored for src/ dir because Chromium sync ' +
'is required. Pass --sync_chromium to force it.')
}
return false
}
if (program.delete_unused_deps) {
if (util.isGitExclusionExists(config.srcDir, 'brave/')) {
args.push('-D')
} else if (!config.isCI) {
Log.warn(
'--delete_unused_deps is ignored because sync has not yet added ' +
'the exclusion for the src/brave/ directory, likely because sync ' +
'has not previously successfully run before.')
}
}
if (program.sync_chromium !== undefined) {
if (!program.sync_chromium) {
Log.warn(
'Chromium needed sync but received the flag to skip performing the ' +
'update. Working directory may not compile correctly.')
return false
} else if (!shouldSyncChromium) {
Log.warn(
'Chromium doesn\'t need sync but received the flag to do it anyway.')
}
}
util.runGClient(args)
util.addGitExclusion(config.srcDir, 'brave/')
util.writeJSON(latestSyncInfoFilePath, expectedSyncInfo)
const postSyncChromiumRef = util.getGitReadableLocalRef(config.srcDir)
Log.status(`Chromium is now at ${postSyncChromiumRef || '[unknown]'}`)
return true
}
function syncBrave(program) {
let args = ['sync', '--nohooks']
const syncWithForce = program.init || program.force
@@ -239,11 +66,11 @@ async function RunCommand() {
}
if (program.init || !fs.existsSync(config.depotToolsDir)) {
maybeInstallDepotTools()
syncUtil.maybeInstallDepotTools()
}
if (program.init || !fs.existsSync(config.defaultGClientFile)) {
buildDefaultGClientConfig(targetOSList, targetArchList)
syncUtil.buildDefaultGClientConfig(targetOSList, targetArchList)
} else if (program.target_os) {
Log.warn(
'--target_os is ignored. If you are attempting to sync with ' +
@@ -259,7 +86,10 @@ async function RunCommand() {
}
Log.progressScope('gclient sync', () => {
const didSyncChromium = syncChromium(program)
const syncWithForce = program.init || program.force
const didSyncChromium = syncUtil.syncChromium(
syncWithForce, program.sync_chromium, program.delete_unused_deps)
if (!didSyncChromium || program.delete_unused_deps) {
// If no Chromium sync was done, run sync inside `brave` to sync Brave DEPS.
syncBrave(program)
package.json
+1
View File
@@ -13,6 +13,7 @@
"sync": "node ./build/commands/scripts/sync.js",
"gclient": "node ./build/commands/scripts/gclient.js",
"build": "node ./build/commands/scripts/commands.js build",
"build_chromium_release": "node ./build/commands/scripts/commands.js build_chromium_release",
"gn_check": "node ./build/commands/scripts/commands.js gn_check",
"versions": "node ./build/commands/scripts/commands.js versions",
"upload": "node ./build/commands/scripts/commands.js upload",