nodejs/deps/v8/tools/builtins-pgo/download_profiles.py

#!/usr/bin/env python3

# Copyright 2023 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.
"""
Download PGO profiles for V8 builtins. The version is pulled from V8's version
file (include/v8-version.h).

See argparse documentation for usage details.
"""

import argparse
from functools import cached_property
import json
import os
import pathlib
import re
import sys

FILE = pathlib.Path(os.path.abspath(__file__))
V8_DIR = FILE.parents[2]
PGO_PROFILE_DIR = V8_DIR / 'tools/builtins-pgo/profiles'

PGO_PROFILE_BUCKET = 'chromium-v8-builtins-pgo'
CHROMIUM_DEPS_V8_REVISION = r"'v8_revision': '([0-9a-f]{40})',"

DEPOT_TOOLS_DEFAULT_PATH = V8_DIR / 'third_party/depot_tools'

VERSION_FILE = V8_DIR / 'include/v8-version.h'
VERSION_RE = r"""#define V8_MAJOR_VERSION (\d+)
#define V8_MINOR_VERSION (\d+)
#define V8_BUILD_NUMBER (\d+)
#define V8_PATCH_LEVEL (\d+)"""


class ProfileDownloader:
  def __init__(self, cmd_args=None):
    self.args = self._parse_args(cmd_args)
    self._import_gsutil()

  def run(self):
    if self.args.action == 'download':
      self._download()
      sys.exit(0)

    if self.args.action == 'validate':
      self._validate()
      sys.exit(0)

    raise AssertionError(f'Invalid action: {args.action}')

  def _parse_args(self, cmd_args):
    parser = argparse.ArgumentParser(
        description=(
            f'Download PGO profiles for V8 builtins generated for the version '
            f'defined in {VERSION_FILE}. If the current checkout has no '
            f'version (i.e. build and patch level are 0 in {VERSION_FILE}), no '
            f'profiles exist and the script returns without errors.'),
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog='\n'.join([
            f'examples:', f'  {FILE.name} download',
            f'  {FILE.name} validate --bucket=chromium-v8-builtins-pgo-staging',
            f'', f'return codes:',
            f'  0 - profiles successfully downloaded or validated',
            f'  1 - unexpected error, see stdout',
            f'  2 - invalid arguments specified, see {FILE.name} --help',
            f'  3 - invalid path to depot_tools provided'
            f'  4 - gsutil was unable to retrieve data from the bucket'
            f'  5 - profiles have been generated for a different revision'
            f'  6 - chromium DEPS file found without v8 revision'
            f'  7 - no chromium DEPS file found'
        ]),
    )

    parser.add_argument(
        'action',
        choices=['download', 'validate'],
        help=(
            'download or validate profiles for the currently checked out '
            'version'
        ),
    )

    parser.add_argument(
        '--version',
        help=('download (or validate) profiles for this version (e.g. '
              '11.0.226.0 or 11.0.226.2), defaults to the version in v8\'s '
              'version file'),
    )

    parser.add_argument(
        '--depot-tools',
        help=('path to depot tools, defaults to V8\'s version in '
              f'{DEPOT_TOOLS_DEFAULT_PATH}.'),
        type=pathlib.Path,
        default=DEPOT_TOOLS_DEFAULT_PATH,
    )

    parser.add_argument(
        '--force',
        help='force download, overwriting existing profiles',
        action='store_true',
    )

    parser.add_argument(
        '--quiet',
        help='run silently, still display errors',
        action='store_true',
    )

    parser.add_argument(
        '--check-v8-revision',
        help='validate profiles are built for chromium\'s V8 revision',
        action='store_true',
    )

    return parser.parse_args(cmd_args)

  def _import_gsutil(self):
    abs_depot_tools_path = os.path.abspath(self.args.depot_tools)
    file = os.path.join(abs_depot_tools_path, 'download_from_google_storage.py')
    if not pathlib.Path(file).is_file():
      message = f'{file} does not exist; check --depot-tools path.'
      self._fail(3, message)

    # Put this path at the beginning of the PATH to give it priority.
    sys.path.insert(0, abs_depot_tools_path)
    globals()['gcs_download'] = __import__('download_from_google_storage')

  @cached_property
  def version(self):
    if self.args.version:
      return self.args.version

    with VERSION_FILE.open() as f:
      version_tuple = re.search(VERSION_RE, f.read()).groups(0)

    version = '.'.join(version_tuple)
    if version_tuple[2] == version_tuple[3] == '0':
      self._log(f'The version file specifies {version}, which has no profiles.')
      sys.exit(0)

    return version

  @cached_property
  def _remote_profile_path(self):
    return f'{PGO_PROFILE_BUCKET}/by-version/{self.version}'

  @cached_property
  def _meta_json_path(self):
    return PGO_PROFILE_DIR / 'meta.json'

  @cached_property
  def _v8_revision(self):
    """If this script is executed within a chromium checkout, return the V8
    revision defined in chromium. Otherwise return None."""

    chromium_deps_file = V8_DIR.parent / 'DEPS'
    if not chromium_deps_file.is_file():
      message = (
          f'File {chromium_deps_file} not found. Verify the parent directory '
          f'of V8 is a valid chromium checkout.'
      )
      self._fail(7, message)

    with chromium_deps_file.open() as f:
      chromum_deps = f.read()

    match = re.search(CHROMIUM_DEPS_V8_REVISION, chromum_deps)
    if not match:
      message = (
          f'No V8 revision can be found in {chromium_deps_file}. Verify this '
          f'is a valid chromium DEPS file including a v8 version entry.'
      )
      self._fail(6, message)

    return match.group(1)

  def _download(self):
    if self._require_download():
      # Wipe profiles directory.
      for file in PGO_PROFILE_DIR.glob('*'):
        if file.name.startswith('.'):
          continue
        file.unlink()

      # Download new profiles.
      path = self._remote_profile_path
      cmd = ['cp', '-R', f'gs://{path}/*', str(PGO_PROFILE_DIR)]
      failure_hint = f'https://storage.googleapis.com/{path} does not exist.'
      self._call_gsutil(cmd, failure_hint)

    # Validate profile revision matches the current V8 revision.
    if self.args.check_v8_revision:
      with self._meta_json_path.open() as meta_json_file:
        meta_json = json.load(meta_json_file)

      if meta_json['revision'] != self._v8_revision:
        message = (
            f'V8 Builtins PGO profiles have been built for '
            f'{meta_json["revision"]}, but this chromium checkout uses '
            f'{self._v8_revision} in its DEPS file. Invalid profiles might '
            f'cause the build to fail or result in performance regressions. '
            f'Select a V8 revision which has up-to-date profiles or build with '
            f'pgo disabled.'
        )
        self._fail(5, message)

  def _require_download(self):
    if self.args.force:
      return True

    if not self._meta_json_path.is_file():
      return True

    with self._meta_json_path.open() as meta_json_file:
      try:
        meta_json = json.load(meta_json_file)
      except json.decoder.JSONDecodeError:
        return True

    if meta_json['version'] != self.version:
      return True

    self._log('Profiles already downloaded, use --force to overwrite.')
    return False

  def _validate(self):
    meta_json = f'{self._remote_profile_path}/meta.json'
    cmd = ['stat', f'gs://{meta_json}']
    failure_hint = (
        f'https://storage.googleapis.com/{meta_json} does not exist. This '
        f'error might be transient. Creating PGO data takes ~20 min after a '
        f'merge to a release branch. You can follow current PGO creation at '
        f'https://ci.chromium.org/ui/p/v8/builders/ci-hp/PGO%20Builder and '
        f'retry the release builder when it\'s ready.')
    self._call_gsutil(cmd, failure_hint)

  def _call_gsutil(self, cmd, failure_hint):
    # Load gsutil from depot tools, and execute command
    gsutil = gcs_download.Gsutil(gcs_download.GSUTIL_DEFAULT_PATH)
    returncode, stdout, stderr = gsutil.check_call(*cmd)
    if returncode != 0:
      self._print_error(['gsutil', *cmd], returncode, stdout, stderr, failure_hint)
      sys.exit(4)

  def _print_error(self, cmd, returncode, stdout, stderr, failure_hint):
    message = [
        'The following command did not succeed:',
        f'  $ {" ".join(cmd)}',
    ]
    sections = [
        ('return code', str(returncode)),
        ('stdout', stdout.strip()),
        ('stderr', stderr.strip()),
        ('hint', failure_hint),
    ]
    for label, output in sections:
      if not output:
        continue
      message += [f'{label}:', "  " + "\n  ".join(output.split("\n"))]

    print('\n'.join(message), file=sys.stderr)

  def _log(self, message):
    if self.args.quiet:
      return
    print(message)

  def _fail(self, returncode, message):
    print(message, file=sys.stderr)
    sys.exit(returncode)


if __name__ == '__main__':
  downloader = ProfileDownloader()
  downloader.run()