const path = require('path') const { spawn, spawnSync } = require('child_process') const config = require('./config') const fs = require('fs-extra') const crypto = require('crypto') const autoGeneratedBraveToChromiumMapping = Object.assign({}, require('./l10nUtil').autoGeneratedBraveToChromiumMapping) const os = require('os') const runGClient = (args, options = {}) => { if (config.gClientVerbose) args.push('--verbose') options.cwd = options.cwd || config.rootDir options = mergeWithDefault(options) options.env.GCLIENT_FILE = config.gClientFile util.run('gclient', args, options) } const mergeWithDefault = (options) => { return Object.assign({}, config.defaultOptions, options) } const util = { run: (cmd, args = [], options = {}) => { console.log(options.cwd + ':', cmd, args.join(' ')) const continueOnFail = options.continueOnFail delete options.continueOnFail const prog = spawnSync(cmd, args, options) if (prog.status !== 0) { if (!continueOnFail) { console.log(prog.stdout && prog.stdout.toString()) console.error(prog.stderr && prog.stderr.toString()) process.exit(1) } } return prog }, runAsync: (cmd, args = [], options = {}) => { let { continueOnFail, verbose, ...cmdOptions } = options if (verbose) { console.log(cmd, args.join(' ')) } return new Promise((resolve, reject) => { const prog = spawn(cmd, args, cmdOptions) let stderr = '' let stdout = '' prog.stderr.on('data', data => { stderr += data }) prog.stdout.on('data', data => { stdout += data }) prog.on('close', statusCode => { const hasFailed = statusCode !== 0 if (verbose && (!hasFailed || continueOnFail)) { console.log(stdout) if (stderr) { console.error(stderr) } } if (hasFailed) { const err = new Error(`Program ${cmd} exited with error code ${statusCode}.`) err.stderr = stderr err.stdout = stdout reject(err) if (!continueOnFail) { console.log(err.message) console.log(stdout) console.error(stderr) process.exit(1) } return } resolve(stdout) }) }) }, runGitAsync: function (repoPath, gitArgs, verbose = false, logError = false) { return util.runAsync('git', gitArgs, { cwd: repoPath, verbose, continueOnFail: true }) .catch(err => { if (logError) { console.error(err.message) console.error(`Git arguments were: ${gitArgs.join(' ')}`) console.log(err.stdout) console.error(err.stderr) } return Promise.reject(err) }) }, buildGClientConfig: () => { function replacer(key, value) { return value; } let solutions = config.projectNames.filter((projectName) => config.projects[projectName].ref).map((projectName) => { let project = config.projects[projectName] return { managed: "%False%", name: project.gclientName, url: project.url, custom_deps: project.custom_deps } }) let cache_dir = process.env.GIT_CACHE_PATH ? ('\ncache_dir = "' + process.env.GIT_CACHE_PATH + '"\n') : '\n' let out = 'solutions = ' + JSON.stringify(solutions, replacer, 2) .replace(/"%None%"/g, "None").replace(/"%False%"/g, "False") + cache_dir if (config.targetOS === 'android') { out = out + "target_os = [ 'android' ]" } else if (config.targetOS === 'ios') { out = out + "target_os = [ 'ios' ]" } fs.writeFileSync(config.defaultGClientFile, out) }, calculateFileChecksum: (filename) => { // adapted from https://github.com/roryrjb/md5-file const BUFFER_SIZE = 8192 const fd = fs.openSync(filename, 'r') const buffer = Buffer.alloc(BUFFER_SIZE) const md5 = crypto.createHash('md5') try { let bytesRead do { bytesRead = fs.readSync(fd, buffer, 0, BUFFER_SIZE) md5.update(buffer.slice(0, bytesRead)) } while (bytesRead === BUFFER_SIZE) } finally { fs.closeSync(fd) } return md5.digest('hex') }, updateBranding: () => { console.log('update branding...') const chromeComponentsDir = path.join(config.srcDir, 'components') const braveComponentsDir = path.join(config.projects['brave-core'].dir, 'components') const chromeAppDir = path.join(config.srcDir, 'chrome', 'app') const braveAppDir = path.join(config.projects['brave-core'].dir, 'app') const chromeBrowserResourcesDir = path.join(config.srcDir, 'chrome', 'browser', 'resources') const braveBrowserResourcesDir = path.join(config.projects['brave-core'].dir, 'browser', 'resources') const braveAppVectorIconsDir = path.join(config.projects['brave-core'].dir, 'vector_icons', 'chrome', 'app') const chromeAndroidDir = path.join(config.srcDir, 'chrome', 'android') const braveAndroidDir = path.join(config.projects['brave-core'].dir, 'android') let fileMap = new Set(); // The following 3 entries we map to the same name, not the chromium equivalent name for copying back autoGeneratedBraveToChromiumMapping[path.join(braveAppDir, 'brave_strings.grd')] = path.join(chromeAppDir, 'brave_strings.grd') autoGeneratedBraveToChromiumMapping[path.join(braveAppDir, 'settings_brave_strings.grdp')] = path.join(chromeAppDir, 'settings_brave_strings.grdp') autoGeneratedBraveToChromiumMapping[path.join(braveComponentsDir, 'components_brave_strings.grd')] = path.join(chromeComponentsDir, 'components_brave_strings.grd') autoGeneratedBraveToChromiumMapping[path.join(braveComponentsDir, 'components_strings.grd')] = path.join(chromeComponentsDir, 'components_strings.grdp') autoGeneratedBraveToChromiumMapping[path.join(braveComponentsDir, 'management_strings.grdp')] = path.join(chromeComponentsDir, 'management_strings.grdp') autoGeneratedBraveToChromiumMapping[path.join(braveComponentsDir, 'password_manager_strings.grdp')] = path.join(chromeComponentsDir, 'password_manager_strings.grdp') autoGeneratedBraveToChromiumMapping[path.join(braveComponentsDir, 'bookmark_bar_strings.grdp')] = path.join(chromeComponentsDir, 'bookmark_bar_strings.grdp') autoGeneratedBraveToChromiumMapping[path.join(braveAndroidDir, 'java', 'strings', 'android_chrome_strings.grd')] = path.join(chromeAndroidDir, 'java', 'strings', 'android_chrome_strings.grd') Object.entries(autoGeneratedBraveToChromiumMapping).forEach(mapping => fileMap.add(mapping)) // Copy xtb files for: // brave/app/resources/chromium_strings*.xtb // brave/app/strings/components_chromium_strings*.xtb // brave/app/resources/generated_resoruces*.xtb fileMap.add([path.join(braveAppDir, 'resources'), path.join(chromeAppDir, 'resources')]) fileMap.add([path.join(braveAppDir, 'strings'), path.join(chromeComponentsDir, 'strings')]) // By overwriting, we don't need to modify some grd files. fileMap.add([path.join(braveAppDir, 'theme', 'brave'), path.join(chromeAppDir, 'theme', 'brave')]) fileMap.add([path.join(braveAppDir, 'theme', 'brave'), path.join(chromeAppDir, 'theme', 'chromium')]) fileMap.add([path.join(braveAppDir, 'theme', 'default_100_percent', 'brave'), path.join(chromeAppDir, 'theme', 'default_100_percent', 'brave')]) fileMap.add([path.join(braveAppDir, 'theme', 'default_200_percent', 'brave'), path.join(chromeAppDir, 'theme', 'default_200_percent', 'brave')]) fileMap.add([path.join(braveAppDir, 'theme', 'default_100_percent', 'brave'), path.join(chromeAppDir, 'theme', 'default_100_percent', 'chromium')]) fileMap.add([path.join(braveAppDir, 'theme', 'default_200_percent', 'brave'), path.join(chromeAppDir, 'theme', 'default_200_percent', 'chromium')]) fileMap.add([path.join(braveAppDir, 'theme', 'default_100_percent', 'common'), path.join(chromeAppDir, 'theme', 'default_100_percent', 'common')]) fileMap.add([path.join(braveAppDir, 'theme', 'default_200_percent', 'common'), path.join(chromeAppDir, 'theme', 'default_200_percent', 'common')]) fileMap.add([path.join(braveComponentsDir, 'resources', 'default_100_percent', 'brave'), path.join(chromeComponentsDir, 'resources', 'default_100_percent', 'chromium')]) fileMap.add([path.join(braveComponentsDir, 'resources', 'default_200_percent', 'brave'), path.join(chromeComponentsDir, 'resources', 'default_200_percent', 'chromium')]) fileMap.add([path.join(braveAppVectorIconsDir, 'vector_icons', 'brave'), path.join(chromeAppDir, 'vector_icons', 'brave')]) // Copy chrome-logo-faded.png for replacing chrome logo of welcome page with brave's on Win8. fileMap.add([path.join(braveBrowserResourcesDir, 'chrome-logo-faded.png'), path.join(chromeBrowserResourcesDir, 'chrome-logo-faded.png')]) // Copy to make our ${branding_path_component}_behaviors.cc fileMap.add([path.join(config.projects['brave-core'].dir, 'chromium_src', 'chrome', 'installer', 'setup', 'brave_behaviors.cc'), path.join(config.srcDir, 'chrome', 'installer', 'setup', 'brave_behaviors.cc')]) for (const [source, output] of fileMap) { if (!fs.existsSync(source)) { console.warn(`Warning: The following file-system entry was not found for copying contents to a chromium destination: ${source}. Consider removing the entry from the file-map, or investigating whether the correct source code reference is checked out.`) continue } let sourceFiles = [] // get all the files if source if a directory if (fs.statSync(source).isDirectory()) { sourceFiles = util.walkSync(source) } else { sourceFiles = [source] } for (const sourceFile of sourceFiles) { let destinationFile = path.join(output, path.relative(source, sourceFile)) // The destination file might be newer when updating chromium so // we check for an exact match on the timestamp. We use seconds instead // of ms because utimesSync doesn't set the times with ms precision if (!fs.existsSync(destinationFile) || Math.floor(new Date(fs.statSync(sourceFile).mtimeMs).getTime() / 1000) != Math.floor(new Date(fs.statSync(destinationFile).mtimeMs).getTime() / 1000)) { fs.copySync(sourceFile, destinationFile) // can't set the date in the past so update the source file // to match the newly copied destionation file const date = fs.statSync(destinationFile).mtime fs.utimesSync(sourceFile, date, date) console.log(sourceFile + ' copied to ' + destinationFile) } } } if (process.platform === 'darwin') { // Copy proper mac app icon for channel to chrome/app/theme/mac/app.icns. // Each channel's app icons are stored in brave/app/theme/$channel/app.icns. // With this copying, we don't need to modify chrome/BUILD.gn for this. const iconSource = path.join(braveAppDir, 'theme', 'brave', 'mac', config.channel, 'app.icns') const iconDest = path.join(chromeAppDir, 'theme', 'brave', 'mac', 'app.icns') if (!fs.existsSync(iconDest) || util.calculateFileChecksum(iconSource) != util.calculateFileChecksum(iconDest)) { console.log('copy app icon') fs.copySync(iconSource, iconDest) } // Copy branding file let branding_file_name = 'BRANDING' if (config.channel) branding_file_name = branding_file_name + '.' + config.channel const brandingSource = path.join(braveAppDir, 'theme', 'brave', branding_file_name) const brandingDest = path.join(chromeAppDir, 'theme', 'brave', 'BRANDING') if (!fs.existsSync(brandingDest) || util.calculateFileChecksum(brandingSource) != util.calculateFileChecksum(brandingDest)) { console.log('copy branding file') fs.copySync(brandingSource, brandingDest) } } if (config.targetOS === 'android') { let androidIconSet = '' if (config.channel === 'development') { androidIconSet = 'res_brave_default' } else if (config.channel === '') { androidIconSet = 'res_brave' } else if (config.channel === 'beta') { androidIconSet = 'res_brave_beta' } else if (config.channel === 'dev') { androidIconSet = 'res_brave_dev' } else if (config.channel === 'nightly') { androidIconSet = 'res_brave_nightly' } const androidIconSource = path.join(braveAppDir, 'theme', 'brave', 'android', androidIconSet) const androidIconDest = path.join(config.srcDir, 'chrome', 'android', 'java', 'res_chromium') const androidResSource = path.join(config.projects['brave-core'].dir, 'android', 'java', 'res') const androidResDest = path.join(config.srcDir, 'chrome', 'android', 'java', 'res') console.log('copy Android app icons') let androidSourceFiles = [] if (fs.statSync(androidIconSource).isDirectory()) { androidSourceFiles = util.walkSync(androidIconSource) } else { androidSourceFiles = [androidIconSource] } for (const androidSourceFile of androidSourceFiles) { let destinationFile = path.join(androidIconDest, path.relative(androidIconSource, androidSourceFile)) if (util.calculateFileChecksum(androidSourceFile) != util.calculateFileChecksum(destinationFile)) { fs.copySync(androidSourceFile, destinationFile) } } console.log('copy Android app resources') androidSourceFiles = [] if (fs.statSync(androidResSource).isDirectory()) { androidSourceFiles = util.walkSync(androidResSource) } else { androidSourceFiles = [androidResSource] } for (const androidSourceFile of androidSourceFiles) { let destinationFile = path.join(androidResDest, path.relative(androidResSource, androidSourceFile)) if (!fs.existsSync(destinationFile) || (util.calculateFileChecksum(androidSourceFile) != util.calculateFileChecksum(destinationFile))) { fs.copySync(androidSourceFile, destinationFile) } } } }, // Chromium compares pre-installed midl files and generated midl files from IDL during the build to check integrity. // Generated files during the build time and upstream pre-installed files are different because we use different IDL file. // So, we should copy our pre-installed files to overwrite upstream pre-installed files. // After checking, pre-installed files are copied to gen dir and they are used to compile. // So, this copying in every build doesn't affect compile performance. updateOmahaMidlFiles: () => { console.log('update omaha midl files...') const srcDir = path.join(config.projects['brave-core'].dir, 'win_build_output', 'midl', 'google_update') const dstDir = path.join(config.srcDir, 'third_party', 'win_build_output', 'midl', 'google_update') fs.copySync(srcDir, dstDir) }, // To build w/o much modification of upstream file, bundling mode is used. To build with this mode, // widevine header file and cdm lib is needed. So, we use fake cdm lib. It only used by gn checking. // Real cdm lib is only donwloaded and installed when user accepts via content settings bubble // because we don't ship cdm lib by default. // Latest version and download url are inserted to cdm header file and brave-core refers it. prepareWidevineCdmBuild: () => { const widevineDir = path.join(config.srcDir, 'third_party', 'widevine', 'cdm', 'linux', 'x64') fs.ensureDirSync(widevineDir) const widevineConfig = { widevineDir, headerFileContent: '', configuredVersion: config.widevineVersion, widevineCdmHeaderFilePath: path.join(widevineDir, 'widevine_cdm_version.h'), fakeWidevineCdmLibFilePath: path.join(widevineDir, 'libwidevinecdm.so'), fakeManifestJson: path.join(widevineDir, 'manifest.json') } widevineConfig.headerFileContent = `#ifndef WIDEVINE_CDM_VERSION_H_ #define WIDEVINE_CDM_VERSION_H_ #define WIDEVINE_CDM_VERSION_STRING \"${widevineConfig.configuredVersion}\" #define WIDEVINE_CDM_DOWNLOAD_URL_STRING \"https://redirector.gvt1.com/edgedl/widevine-cdm/${widevineConfig.configuredVersion}-linux-x64.zip\" #endif // WIDEVINE_CDM_VERSION_H_` // If version file or fake lib file aren't existed, create them. if (!fs.existsSync(widevineConfig.widevineCdmHeaderFilePath) || !fs.existsSync(widevineConfig.fakeWidevineCdmLibFilePath) || !fs.existsSync(widevineConfig.fakeManifestJson)) { util.doPrepareWidevineCdmBuild(widevineConfig) return } // Check version file has latest version. If not create it. // This can prevent unnecessary build by touched version file. const installedHeaderFileContent = fs.readFileSync(widevineConfig.widevineCdmHeaderFilePath, 'utf8') if (installedHeaderFileContent !== widevineConfig.headerFileContent) { console.log("Current version file includes different version with latest") util.doPrepareWidevineCdmBuild(widevineConfig) } }, doPrepareWidevineCdmBuild: (widevineConfig) => { console.log('prepare widevine cdm build in linux') fs.writeFileSync(widevineConfig.widevineCdmHeaderFilePath, widevineConfig.headerFileContent) fs.writeFileSync(widevineConfig.fakeWidevineCdmLibFilePath, '') fs.writeFileSync(widevineConfig.fakeManifestJson, '{}') // During the create_dist, /usr/lib/rpm/elfdeps requires that binaries have an exectuable bit set. fs.chmodSync(widevineConfig.fakeWidevineCdmLibFilePath, 0o755) }, signApp: (options = config.defaultOptions) => { console.log('signing ...') util.run('ninja', ['-C', config.outputDir, config.signTarget], options) }, signWinBinaries: () => { if (config.buildConfig !== 'Release' || config.skip_signing) { console.log('binary signing is skipped because this is not official build or --skip_signing is used') return } if (process.env.CERT === undefined || process.env.SIGNTOOL_ARGS === undefined) { console.log('binary signing is skipped because of missing env vars') return } console.log('signing win binaries...') // Copy & sign only binaries for widevine sig file generation. // With this, create_dist doesn't trigger rebuild because original binaries is not modified. const dir = path.join(config.outputDir, 'signed_binaries') if (!fs.existsSync(dir)) fs.mkdirSync(dir); fs.copySync(path.join(config.outputDir, 'brave.exe'), path.join(dir, 'brave.exe')); fs.copySync(path.join(config.outputDir, 'chrome.dll'), path.join(dir, 'chrome.dll')); fs.copySync(path.join(config.outputDir, 'chrome_child.dll'), path.join(dir, 'chrome_child.dll')); const core_dir = config.projects['brave-core'].dir util.run('python', [path.join(core_dir, 'script', 'sign_binaries.py'), '--build_dir=' + dir]) }, generateWidevineSigFiles: () => { const cert = config.sign_widevine_cert const key = config.sign_widevine_key const passwd = config.sign_widevine_passwd const sig_generator = config.signature_generator let src_dir = path.join(config.outputDir, 'signed_binaries') if (config.skip_signing || process.env.CERT === undefined || process.env.SIGNTOOL_ARGS === undefined) src_dir = config.outputDir console.log('generate Widevine sig files...') util.run('python', [sig_generator, '--input_file=' + path.join(src_dir, 'brave.exe'), '--flags=1', '--certificate=' + cert, '--private_key=' + key, '--output_file=' + path.join(config.outputDir, 'brave.exe.sig'), '--private_key_passphrase=' + passwd]) util.run('python', [sig_generator, '--input_file=' + path.join(src_dir, 'chrome.dll'), '--flags=0', '--certificate=' + cert, '--private_key=' + key, '--output_file=' + path.join(config.outputDir, 'chrome.dll.sig'), '--private_key_passphrase=' + passwd]) util.run('python', [sig_generator, '--input_file=' + path.join(src_dir, 'chrome_child.dll'), '--flags=0', '--certificate=' + cert, '--private_key=' + key, '--output_file=' + path.join(config.outputDir, 'chrome_child.dll.sig'), '--private_key_passphrase=' + passwd]) }, buildTarget: (options = config.defaultOptions) => { console.log('building ' + config.buildTarget + '...') if (process.platform === 'win32') util.updateOmahaMidlFiles() if (process.platform === 'linux') util.prepareWidevineCdmBuild() let num_compile_failure = 1 if (config.ignore_compile_failure) num_compile_failure = 0 const args = util.buildArgsToString(config.buildArgs()) util.run('gn', ['gen', config.outputDir, '--args="' + args + '"'], options) let ninjaOpts = [ '-C', config.outputDir, config.buildTarget, '-k', num_compile_failure, ...config.extraNinjaOpts ] util.run('ninja', ninjaOpts, options) }, generateXcodeWorkspace: (options = config.defaultOptions) => { console.log('generating Xcode workspace for "' + config.xcode_gen_target + '"...') const args = util.buildArgsToString(config.buildArgs()) const genScript = path.join(config.rootDir, 'vendor', 'gn-project-generators', 'xcode.py') const genArgs = [ 'gen', config.outputDir + "_Xcode", '--args="' + args + '"', '--ide=json', '--json-ide-script="' + genScript + '"', '--filters="' + config.xcode_gen_target + '"' ] util.run('gn', genArgs, options) }, lint: (options = {}) => { if (!options.base) { options.base = 'origin/master'; } let cmd_options = config.defaultOptions cmd_options.cwd = config.projects['brave-core'].dir util.run('vpython', [path.join(config.rootDir, 'scripts', 'lint.py'), '--project_root=' + config.srcDir, '--base_branch=' + options.base], cmd_options) }, fixDepotTools: (options = {}) => { if (process.platform !== 'win32') { util.run('git', ['-C', config.depotToolsDir, 'clean', '-fxd'], options) util.run('git', ['-C', config.depotToolsDir, 'reset', '--hard', 'HEAD'], options) return } // On Windows: // When depot_tools are already installed they redirect git to their own // version which resides in a bootstrap-*_bin directory. So when we try to // do git clean -fxd we fail because the git executable is in use in that // directory. Get around that by using regular git. let git_exes = util.run('where', ['git'], {shell: true}) let git_exe = '"' + git_exes.stdout.toString().split(os.EOL)[0] + '"' if (git_exe === '""') git_exe = 'git' util.run(git_exe, ['-C', config.depotToolsDir, 'clean', '-fxd'], options) util.run(git_exe, ['-C', config.depotToolsDir, 'reset', '--hard', 'HEAD'], options) // Get around the error in updating depot_tools on windows due to pylint.bat // file transitioning from untracked to a commited file. When // update_depot_tools script tries to use git rebase it errors out. This is // already fixed upstream, but we need a workaround for // now. See https://bugs.chromium.org/p/chromium/issues/detail?id=996359 // The commit id in git merge-base command below is when pylint.bat was // added to git. let cmd_options = Object.assign({}, options) cmd_options.continueOnFail = true let is_fixed = util.run('git', ['-C', config.depotToolsDir, 'merge-base', '--is-ancestor', '53297790de09e48c91678367b48528afbc9f71c1', 'HEAD'], cmd_options) // If merge-base succeeds the exit code is 0. if (!is_fixed.status) return console.log("Manually updating depot_tools as a workaround for https://crbug.com/996359") util.run('git', ['-C', config.depotToolsDir, 'fetch', 'origin'], options) util.run('git', ['-C', config.depotToolsDir, 'checkout', 'origin/master'], options) util.run('git', ['-C', config.depotToolsDir, 'reset', '--hard', 'origin/master'], options) }, submoduleSync: (options = {}) => { if (!options.cwd) options.cwd = config.rootDir // default cwd `./src` may not exist yet options = mergeWithDefault(options) util.run('git', ['submodule', 'sync'], options) util.run('git', ['submodule', 'update', '--init', '--recursive'], options) util.fixDepotTools(options) }, gclientSync: (reset = false, options = {}) => { let args = ['sync', '--force', '--nohooks', '--with_branch_heads', '--with_tags'] if (reset) args.push('--upstream') runGClient(args, options) }, gclientRunhooks: (options = {}) => { runGClient(['runhooks'], options) }, fetch: (gitRepoPath) => { return util.runGitAsync(gitRepoPath, ['fetch', '--all', '--tags']) }, setGitVersion: async (gitRepoPath, version, alwaysReset = false) => { await util.runGitAsync(gitRepoPath, ['clean', '-f']) let shouldReset = alwaysReset if (!shouldReset) { const headSHA = await util.runGitAsync(gitRepoPath, ['rev-parse', 'HEAD']) const targetSHA = await util.runGitAsync(gitRepoPath, ['rev-parse', version]) shouldReset = (headSHA !== targetSHA) } if (shouldReset) { await util.runGitAsync(gitRepoPath, ['reset', '--hard', version]) } return shouldReset }, buildArgsToString: (buildArgs) => { let args = '' for (let arg in buildArgs) { let val = buildArgs[arg] if (typeof val === 'string') { val = '"' + val + '"' } else { val = JSON.stringify(val) } args += arg + '=' + val + ' ' } return args.replace(/"/g,'\\"') }, walkSync: (dir, filter = null, filelist = []) => { fs.readdirSync(dir).forEach(file => { if (fs.statSync(path.join(dir, file)).isDirectory()) { filelist = util.walkSync(path.join(dir, file), filter, filelist) } else if (!filter || filter.call(null, file)) { filelist = filelist.concat(path.join(dir, file)) } }) return filelist } } module.exports = util