Revert "Improve npm run sync logic to support v8 patching."

This commit is contained in:
Brian Johnson
2022-08-22 12:29:22 -07:00
committed by GitHub
parent 0c808c3a27
commit beb67681d9
6 changed files with 149 additions and 258 deletions
-8
View File
@@ -1,8 +0,0 @@
solutions = [
{
"managed": False,
"name": ".",
# We do not use gclient to manage brave-core, so this not actually get used.
"url": "https://github.com/brave/brave-core.git"
}
]
-1
View File
@@ -2,7 +2,6 @@
!/vendor/bat-native-ledger
!/vendor/bat-native-ads
.DS_Store
.gclient_*
.tags*
/.idea/
/components/brave_new_tab_ui/data/LICENSE
+1
View File
@@ -928,6 +928,7 @@ Object.defineProperty(Config.prototype, 'defaultOptions', {
env = this.addPythonPathToEnv(env, path.join(this.srcDir, 'brave', 'vendor', 'requests'))
env = this.addPythonPathToEnv(env, path.join(this.srcDir, 'build'))
env = this.addPythonPathToEnv(env, path.join(this.srcDir, 'third_party', 'depot_tools'))
env.GCLIENT_FILE = this.gClientFile
env.DEPOT_TOOLS_WIN_TOOLCHAIN = '0'
env.PYTHONUNBUFFERED = '1'
env.TARGET_ARCH = this.gypTargetArch // for brave scripts
+3 -3
View File
@@ -16,7 +16,7 @@ process.stdout.on('resize', setLineLength)
const progressStyle = chalk.bold.inverse
const statusStyle = chalk.green.italic
const warningStyle = chalk.black.bgYellow
const warningStyle = chalk.black.bold.bgYellow
const cmdDirStyle = chalk.blue
const cmdCmdStyle = chalk.green
@@ -30,11 +30,11 @@ function status(message) {
console.log(statusStyle(message))
}
function error(message) {
function error (message) {
console.error(progressStyle(message))
}
function warn(message) {
function warn (message) {
console.error(warningStyle(message))
}
+128 -51
View File
@@ -1,4 +1,5 @@
const path = require('path')
const chalk = require('chalk')
const { spawn, spawnSync } = require('child_process')
const config = require('./config')
const fs = require('fs-extra')
@@ -7,6 +8,18 @@ const l10nUtil = require('./l10nUtil')
const Log = require('./sync/logging')
const assert = require('assert')
const runGClient = (args, options = {}) => {
if (config.gClientVerbose) args.push('--verbose')
options.cwd = options.cwd || config.rootDir
options = mergeWithDefault(options)
options.env.GCLIENT_FILE = config.gClientFile
util.run('gclient', args, options)
}
const mergeWithDefault = (options) => {
return Object.assign({}, config.defaultOptions, options)
}
async function applyPatches() {
const GitPatcher = require('./gitPatcher')
Log.progress('Applying patches...')
@@ -76,9 +89,6 @@ const getAdditionalGenLocation = () => {
}
const util = {
mergeOptionsWithDefault: (options) => {
return Object.assign({}, config.defaultOptions, options)
},
runProcess: (cmd, args = [], options = {}) => {
Log.command(options.cwd, cmd, args)
@@ -167,6 +177,50 @@ const util = {
return util.runGit(repoDir, ['log', '-n', '1', '--pretty=format:%h%d'], true)
},
buildGClientConfig: () => {
function replacer(key, value) {
return value;
}
const solutions = [
{
managed: "%False%",
name: "src",
url: config.chromiumRepo,
custom_deps: {
"src/third_party/WebKit/LayoutTests": "%None%",
"src/chrome_frame/tools/test/reference_build/chrome": "%None%",
"src/chrome_frame/tools/test/reference_build/chrome_win": "%None%",
"src/chrome/tools/test/reference_build/chrome": "%None%",
"src/chrome/tools/test/reference_build/chrome_linux": "%None%",
"src/chrome/tools/test/reference_build/chrome_mac": "%None%",
"src/chrome/tools/test/reference_build/chrome_win": "%None%"
},
custom_vars: {
"checkout_pgo_profiles": config.isBraveReleaseBuild() ? "%True%" : "%False%"
}
},
{
managed: "%False%",
name: "src/brave",
// We do not use gclient to manage brave-core, so this should
// not actually get used.
url: 'https://github.com/brave/brave-core.git'
}
]
let cache_dir = process.env.GIT_CACHE_PATH ? ('\ncache_dir = "' + process.env.GIT_CACHE_PATH + '"\n') : '\n'
let out = 'solutions = ' + JSON.stringify(solutions, replacer, 2)
.replace(/"%None%"/g, "None").replace(/"%False%"/g, "False").replace(/"%True%"/g, "True") + cache_dir
if (config.targetOS) {
out = out + "target_os = [ '" + config.targetOS + "' ]"
}
fs.writeFileSync(config.defaultGClientFile, out)
},
calculateFileChecksum: (filename) => {
// adapted from https://github.com/kodie/md5-file
const BUFFER_SIZE = 8192
@@ -558,9 +612,7 @@ const util = {
const buildArgsStr = util.buildArgsToString(gnArgs)
util.run('gn', ['gen', config.nativeRedirectCCDir, '--args="' + buildArgsStr + '"'], options)
util.buildTarget(
'brave/tools/redirect_cc',
util.mergeOptionsWithDefault({outputDir: config.nativeRedirectCCDir}))
util.buildTarget('brave/tools/redirect_cc', mergeWithDefault({outputDir: config.nativeRedirectCCDir}))
},
runGnGen: (options) => {
@@ -674,7 +726,9 @@ const util = {
if (!options.base) {
options.base = 'origin/master'
}
const cmd_options = util.mergeOptionsWithDefault({cwd: config.braveCoreDir})
let cmd_options = config.defaultOptions
cmd_options.cwd = config.braveCoreDir
cmd_options = mergeWithDefault(cmd_options)
util.run('vpython', [path.join(config.braveCoreDir, 'build', 'commands', 'scripts', 'lint.py'),
'--project_root=' + config.srcDir,
'--base_branch=' + options.base], cmd_options)
@@ -688,7 +742,9 @@ const util = {
// 'gerrit.host' from their brave checkout.
util.runGit(
config.braveCoreDir, ['config', '--unset-all', 'gerrit.host'], true)
const cmd_options = util.mergeOptionsWithDefault({cwd: config.braveCoreDir})
let cmd_options = config.defaultOptions
cmd_options.cwd = config.braveCoreDir
cmd_options = mergeWithDefault(cmd_options)
cmd = 'git'
args = ['cl', 'presubmit', options.base, '--force']
if (options.all)
@@ -702,7 +758,9 @@ const util = {
if (!options.base) {
options.base = 'origin/master'
}
const cmd_options = util.mergeOptionsWithDefault({cwd: config.braveCoreDir})
let cmd_options = config.defaultOptions
cmd_options.cwd = config.braveCoreDir
cmd_options = mergeWithDefault(cmd_options)
cmd = 'git'
args = ['cl', 'format', '--upstream=' + options.base]
if (options.full)
@@ -724,16 +782,68 @@ const util = {
util.run('python3', [path.join(config.srcDir, 'tools', 'git', 'mass-rename.py')], cmd_options)
},
runGClient: (args, options, gClientFile) => {
if (config.gClientVerbose) {
args.push('--verbose')
shouldUpdateChromium: (chromiumRef = config.getProjectRef('chrome')) => {
const headSHA = util.runGit(config.srcDir, ['rev-parse', 'HEAD'], true)
const targetSHA = util.runGit(config.srcDir, ['rev-parse', chromiumRef], true)
const needsUpdate = ((targetSHA !== headSHA) || (!headSHA && !targetSHA))
if (needsUpdate) {
const currentRef = util.getGitReadableLocalRef(config.srcDir)
console.log(`Chromium repo ${chalk.blue.bold('needs update')}. Target is ${chalk.italic(chromiumRef)} at commit ${targetSHA || '[missing]'} but current commit is ${chalk.italic(currentRef || '[unknown]')} at commit ${chalk.inverse(headSHA || '[missing]')}.`)
} else {
console.log(chalk.green.bold(`Chromium repo does not need update as it is already ${chalk.italic(chromiumRef)} at commit ${targetSHA || '[missing]'}.`))
}
options.cwd = options.cwd || config.rootDir
options = util.mergeOptionsWithDefault(options)
if (gClientFile) {
options.env.GCLIENT_FILE = gClientFile
return needsUpdate
},
gclientSync: (forceReset = false, cleanup = false, shouldCheckChromiumVersion = true, options = {}) => {
let reset = forceReset
// base args
const initialArgs = ['sync', '--nohooks']
const chromiumArgs = ['--revision', 'src@' + config.getProjectRef('chrome')]
const resetArgs = ['--reset', '--with_tags', '--with_branch_heads', '--upstream']
let args = [...initialArgs]
let didUpdateChromium = false
if (!shouldCheckChromiumVersion) {
const chromiumNeedsUpdate = util.shouldUpdateChromium()
if (chromiumNeedsUpdate) {
console.warn(chalk.yellow.bold('Chromium needed update but received the flag to skip performing the update. Working directory may not compile correctly.'))
}
} else if (forceReset || util.shouldUpdateChromium()) {
args = [...args, ...chromiumArgs]
reset = true
didUpdateChromium = true
}
util.run('gclient', args, options)
if (forceReset) {
args = args.concat(['--force'])
if (cleanup) {
// temporarily ignored until we can figure out how not to delete src/brave in the process
// args = args.concat(['-D'])
}
}
if (reset) {
args = [...args, ...resetArgs]
}
runGClient(args, options)
return {
didUpdateChromium
}
},
gclientRunhooks: (options = {}) => {
Log.progress('Running gclient hooks...')
runGClient(['runhooks'], options)
Log.progress('Done running gclient hooks.')
},
runGClient: (args, options) => {
runGClient(args, options)
},
applyPatches: () => {
@@ -771,40 +881,7 @@ const util = {
if (process.platform === 'win32')
input += '.exe'
return input
},
isGitExclusionExists: (dir, exclusion) => {
const excludeFile = path.join(dir, '.git', 'info', 'exclude')
if (!fs.existsSync(excludeFile)) {
return false
}
const lines = fs.readFileSync(excludeFile).toString().split(/\r?\n/)
for (const line of lines) {
if (line === exclusion) {
return true
}
}
return false
},
addGitExclusion: (dir, exclusion) => {
if (util.isGitExclusionExists(dir, exclusion)) {
return
}
const excludeFile = path.join(dir, '.git', 'info', 'exclude')
fs.appendFileSync(excludeFile, '\n' + exclusion)
},
readJSON: (file, default_value={}) => {
if (!fs.existsSync(file)) {
return default_value
}
return fs.readJSONSync(file)
},
writeJSON: (file, value) => {
return fs.writeJSONSync(file, value, {spaces: 2})
},
}
}
module.exports = util
+17 -195
View File
@@ -9,7 +9,6 @@ const path = require('path')
const config = require('../lib/config')
const util = require('../lib/util')
const Log = require('../lib/sync/logging')
const chalk = require('chalk')
program
.version(process.env.npm_package_version)
@@ -20,24 +19,16 @@ program
.option('--target_android_base <target_android_base>', 'target Android OS level for apk or aab (classic, modern, mono)')
.option('--init', 'initialize all dependencies')
.option('--force', 'force reset all projects to origin/ref')
.option('--sync_chromium [arg]', 'force or skip chromium sync (true/false/1/0)', JSON.parse)
.option('--ignore_chromium', 'do not update chromium version even if it is stale [deprecated, use --sync_chromium=false]')
.option('--sync_chromium_and_delete_unused_deps', 'force chromium sync and delete from the working copy any dependencies that have been removed since the last sync')
.option('--ignore_chromium', 'do not update chromium version even if it is stale')
.option('--nohooks', 'Do not run hooks after updating')
function maybeInstallDepotTools() {
const maybeInstallDepotTools = (options = config.defaultOptions) => {
options.cwd = config.braveCoreDir
if (!fs.existsSync(config.depotToolsDir)) {
Log.progress('Install Depot Tools...')
fs.mkdirSync(config.depotToolsDir)
const options = util.mergeOptionsWithDefault({cwd: config.depotToolsDir})
util.run(
'git',
[
'clone',
'https://chromium.googlesource.com/chromium/tools/depot_tools.git',
'.'
],
options)
util.run('git', ['-C', config.depotToolsDir, 'clone', 'https://chromium.googlesource.com/chromium/tools/depot_tools.git', '.'], options)
Log.progress('Done Depot Tools...')
}
@@ -50,209 +41,40 @@ function maybeInstallDepotTools() {
'is-googler': false,
'version': 3,
'countdown': 10,
'opt-in': false,
'opt-in': false
};
fs.writeFileSync(ninjaLogCfgPath, JSON.stringify(ninjaLogCfgConfig))
}
}
function toGClientConfigItem(name, value, pretty = true) {
// Convert value to json and replace "%True%" -> True, "%False%" -> False,
// "%None%" -> None.
const pythonLikeValue =
JSON.stringify(value, null, pretty ? 2 : 0).replace(/"%(.*?)%"/gm, '$1')
return `${name} = ${pythonLikeValue}\n`
}
function buildDefaultGClientConfig() {
let out = toGClientConfigItem('solutions', [
{
managed: '%False%',
name: 'src',
url: config.chromiumRepo,
custom_deps: {
'src/third_party/WebKit/LayoutTests': '%None%',
'src/chrome_frame/tools/test/reference_build/chrome': '%None%',
'src/chrome_frame/tools/test/reference_build/chrome_win': '%None%',
'src/chrome/tools/test/reference_build/chrome': '%None%',
'src/chrome/tools/test/reference_build/chrome_linux': '%None%',
'src/chrome/tools/test/reference_build/chrome_mac': '%None%',
'src/chrome/tools/test/reference_build/chrome_win': '%None%'
},
custom_vars: {
'checkout_pgo_profiles': config.isBraveReleaseBuild() ? '%True%' :
'%False%'
}
},
{
managed: '%False%',
name: 'src/brave',
// We do not use gclient to manage brave-core, so this should not
// actually get used.
url: 'https://github.com/brave/brave-core.git'
}
])
if (process.env.GIT_CACHE_PATH) {
out += toGClientConfigItem('cache_dir', process.env.GIT_CACHE_PATH)
}
if (config.targetOS) {
out += toGClientConfigItem('target_os', [config.targetOS], false)
}
fs.writeFileSync(config.defaultGClientFile, out)
}
function shouldUpdateChromium(latestSuccessfulSyncInfo, expectedSuccessfulSyncInfo) {
const chromiumRef = expectedSuccessfulSyncInfo.chromiumRef
const headSHA = util.runGit(config.srcDir, ['rev-parse', 'HEAD'], true)
const targetSHA = util.runGit(config.srcDir, ['rev-parse', chromiumRef], true)
const needsUpdate = targetSHA !== headSHA || (!headSHA && !targetSHA) ||
JSON.stringify(latestSuccessfulSyncInfo) !==
JSON.stringify(expectedSuccessfulSyncInfo)
if (needsUpdate) {
const currentRef = util.getGitReadableLocalRef(config.srcDir)
console.log(
`Chromium repo ${chalk.blue.bold('needs sync')}.\n target is ${
chalk.italic(chromiumRef)} at commit ${
targetSHA || '[missing]'}\n current commit is ${
chalk.italic(currentRef || '[unknown]')} at commit ${
chalk.inverse(
headSHA || '[missing]')}\n latest successful sync is ${
JSON.stringify(latestSuccessfulSyncInfo, null, 4)}`)
} else {
console.log(
chalk.green.bold(`Chromium repo does not need sync as it is already ${
chalk.italic(
chromiumRef)} at commit ${targetSHA || '[missing]'}.`))
}
return needsUpdate
}
function syncChromium(program) {
const requiredChromiumRef = config.getProjectRef('chrome')
let args = [
'sync', '--nohooks', '--reset', '--revision',
'src@' + requiredChromiumRef, '--with_tags',
'--with_branch_heads', '--upstream'
];
const syncWithForce = program.init || program.force
if (syncWithForce) {
args.push('--force')
}
if (program.sync_chromium_and_delete_unused_deps) {
if (util.isGitExclusionExists(config.srcDir, 'brave/')) {
args.push('-D')
} else {
Log.warn(
'--sync_chromium_and_delete_unused_deps was specified but cannot ' +
'be used to remove old Chromium deps as sync has not yet added the ' +
'exclusion for the src/brave/ directory, likely because sync has ' +
'not previously successfully run before.')
}
}
const latestSuccessfulSyncFilePath =
path.join(config.rootDir, '.brave_latest_successful_sync.json')
const latestSuccessfulSyncInfo = util.readJSON(latestSuccessfulSyncFilePath)
const expectedSuccessfulSyncInfo = {
chromiumRef: requiredChromiumRef,
gClientTimestamp: fs.statSync(config.gClientFile).mtimeMs.toString(),
}
const chromiumNeedsUpdate =
shouldUpdateChromium(latestSuccessfulSyncInfo, expectedSuccessfulSyncInfo)
const shouldSyncChromium =
chromiumNeedsUpdate || syncWithForce || program.sync_chromium
if (!shouldSyncChromium) {
return false
}
if (program.sync_chromium !== undefined) {
if (!program.sync_chromium) {
Log.warn(
'Chromium needed sync but received the flag to skip performing the ' +
'update. Working directory may not compile correctly.')
return false
} else if (!chromiumNeedsUpdate && !syncWithForce) {
Log.warn(
'Chromium doesn\'t need sync but received the flag to do it anyway.')
}
}
util.runGClient(args, {cwd: config.rootDir}, config.gClientFile)
util.addGitExclusion(config.srcDir, 'brave/')
util.writeJSON(latestSuccessfulSyncFilePath, expectedSuccessfulSyncInfo)
const postSyncChromiumRef = util.getGitReadableLocalRef(config.srcDir)
Log.status(`Chromium is now at ${postSyncChromiumRef || '[unknown]'}`)
return true
}
function syncBrave(program) {
let args = ['sync', '--nohooks']
const syncWithForce = program.init || program.force
if (syncWithForce) {
args.push('--force')
}
// Don't pass gClientFile here, let gclient find it automatically, which
// should be brave/.gclient.
util.runGClient(args, {cwd: config.braveCoreDir}, null)
}
function gclientRunhooks() {
util.runGClient(['runhooks'], {cwd: config.rootDir}, config.gClientFile)
}
async function RunCommand () {
program.parse(process.argv)
config.update(program)
if (program.ignore_chromium) {
Log.warn(
'--ignore_chromium is deprecated, please replpace with ' +
'--sync_chromium=false')
program.sync_chromium = false
}
if (program.sync_chromium_and_delete_unused_deps) {
program.sync_chromium = true
if (program.all || program.run_hooks || program.run_sync) {
Log.warn('--all, --run_hooks and --run_sync are deprecated. Will behave as if flag was not passed. Please update your command to `npm run sync` in the future.')
}
if (program.init || !fs.existsSync(config.depotToolsDir)) {
maybeInstallDepotTools()
}
if (program.init || !fs.existsSync(config.defaultGClientFile)) {
buildDefaultGClientConfig()
} else if (program.target_os) {
Log.warn(
'--target_os is ignored. If you are attempting to sync with ' +
'a different target_os argument from that used originally via init ' +
'(and specified in the .gclient file), then you will likely not end ' +
'up with the correct dependency projects. Specify new target_os ' +
'values with --init, or edit .gclient manually before running sync ' +
'again.')
if (program.init) {
util.buildGClientConfig()
}
Log.progress('Running gclient sync...')
const didSyncChromium = syncChromium(program)
if (!didSyncChromium) {
// If no Chromium sync was done, run sync inside `brave` to sync Brave DEPS.
syncBrave(program)
const result = util.gclientSync(program.init || program.force, program.init, !program.ignore_chromium)
if (result.didUpdateChromium) {
const postSyncChromiumRef = util.getGitReadableLocalRef(config.srcDir)
Log.status(`Chromium is now at ${postSyncChromiumRef || '[unknown]'}`)
}
Log.progress('...gclient sync done.')
Log.progress('...gclient sync done')
await util.applyPatches()
if (!program.nohooks) {
// Run hooks for the root .gclient, this will include Chromium and Brave
// hooks. Don't cache the result, just always rerun this step, because it's
// pretty quick in a no-op scenario.
Log.progress('Running gclient runhooks...')
gclientRunhooks()
Log.progress('...gclient runhooks done.')
util.gclientRunhooks()
}
}