1berry/blender
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tools: add --verbose argument for code_clean.py

Browse Source 
Support `--verbose compile,edit_actions` (one or both can be selected),
to output the compiler output of each edit and the text (before/after).

This replaces existing hard-coded values.

Other minor changes:

- Print the file-path for each edit for better context.
- Print skipped edits when multiple edits are passed in so other
  edits might be applied separately are listed.
This commit is contained in:
Campbell Barton 2023-07-21 11:03:11 +10:00
parent b29a4cdcfc
commit 22354e8f84
1 changed files with 128 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
tools/utils_maintenance/code_clean.py
View File

@ -33,23 +33,28 @@ from typing import (
# List of (source_file, all_arguments) # List of (source_file, all_arguments)
ProcessedCommands = List[Tuple[str, str]] ProcessedCommands = List[Tuple[str, str]]
VERBOSE = False
# Print the output of the compiler (_very_ noisy, only useful for troubleshooting compiler issues).
VERBOSE_COMPILER = False
# Print the result of each attempted edit:
#
# - Causes code not to compile.
# - Compiles but changes the resulting behavior.
# - Succeeds.
VERBOSE_EDIT_ACTION = False
BASE_DIR = os.path.abspath(os.path.dirname(__file__)) BASE_DIR = os.path.abspath(os.path.dirname(__file__))
SOURCE_DIR = os.path.normpath(os.path.join(BASE_DIR, "..", "..")) SOURCE_DIR = os.path.normpath(os.path.join(BASE_DIR, "..", ".."))
# (id: doc-string) pairs.
VERBOSE_INFO = [
(
"compile", (
"Print the compiler output (noisy).\n"
"Try setting '--jobs=1' for usable output.\n"
),
),
(
"edit_actions", (
"Print the result of each attempted edit, useful for troubleshooting:\n"
"- Causes code not to compile.\n"
"- Compiles but changes the resulting behavior.\n"
"- Succeeds.\n"
),
)
]
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Generic Constants # Generic Constants
@ -181,8 +186,14 @@ def text_matching_bracket_backward(
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Execution Wrappers # Execution Wrappers
def run(args: Sequence[str], *, cwd: Optional[str], quiet: bool) -> int: def run(
if VERBOSE_COMPILER and not quiet: args: Sequence[str],
*,
cwd: Optional[str],
quiet: bool,
verbose_compile: bool,
) -> int:
if verbose_compile and not quiet:
out = sys.stdout.fileno() out = sys.stdout.fileno()
else: else:
out = subprocess.DEVNULL out = subprocess.DEVNULL
@ -1278,8 +1289,11 @@ def test_edit(
build_cwd: Optional[str], build_cwd: Optional[str],
data: str, data: str,
data_test: str, data_test: str,
keep_edits: bool = True, *,
expect_failure: bool = False, keep_edits: bool,
expect_failure: bool,
verbose_compile: bool,
verbose_edit_actions: bool,
) -> bool: ) -> bool:
""" """
Return true if `data_test` has the same object output as `data`. Return true if `data_test` has the same object output as `data`.
@ -1290,7 +1304,7 @@ def test_edit(
with open(source, 'w', encoding='utf-8') as fh: with open(source, 'w', encoding='utf-8') as fh:
fh.write(data_test) fh.write(data_test)
ret = run(build_args, cwd=build_cwd, quiet=expect_failure) ret = run(build_args, cwd=build_cwd, quiet=expect_failure, verbose_compile=verbose_compile)
if ret == 0: if ret == 0:
output_bytes_test = file_as_bytes(output) output_bytes_test = file_as_bytes(output)
if (output_bytes is None) or (file_as_bytes(output) == output_bytes): if (output_bytes is None) or (file_as_bytes(output) == output_bytes):
@ -1299,11 +1313,11 @@ def test_edit(
fh.write(data) fh.write(data)
return True return True
else: else:
if VERBOSE_EDIT_ACTION: if verbose_edit_actions:
print("Changed code, skip...", hex(hash(output_bytes)), hex(hash(output_bytes_test))) print("Changed code, skip...", hex(hash(output_bytes)), hex(hash(output_bytes_test)))
else: else:
if not expect_failure: if not expect_failure:
if VERBOSE_EDIT_ACTION: if verbose_edit_actions:
print("Failed to compile, skip...") print("Failed to compile, skip...")
with open(source, 'w', encoding='utf-8') as fh: with open(source, 'w', encoding='utf-8') as fh:
@ -1361,7 +1375,7 @@ def edit_group_compatible(edits: Sequence[str]) -> Sequence[Sequence[str]]:
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Accept / Reject Edits # Accept / Reject Edits
def apply_edit(data: str, text_to_replace: str, start: int, end: int, *, verbose: bool) -> str: def apply_edit(source_relative: str, data: str, text_to_replace: str, start: int, end: int, *, verbose: bool) -> str:
if verbose: if verbose:
line_before = line_from_span(data, start, end) line_before = line_from_span(data, start, end)
@ -1372,7 +1386,7 @@ def apply_edit(data: str, text_to_replace: str, start: int, end: int, *, verbose
line_after = line_from_span(data, start, end) line_after = line_from_span(data, start, end)
print("") print("")
print("Testing edit:") print("Testing edit:", source_relative)
print(line_before) print(line_before)
print(line_after) print(line_after)
@ -1385,9 +1399,14 @@ def wash_source_with_edit(
build_args: Sequence[str], build_args: Sequence[str],
build_cwd: Optional[str], build_cwd: Optional[str],
skip_test: bool, skip_test: bool,
verbose_compile: bool,
verbose_edit_actions: bool,
shared_edit_data: Any, shared_edit_data: Any,
edit_to_apply: str, edit_to_apply: str,
) -> None: ) -> None:
# For less verbose printing, strip the prefix.
source_relative = os.path.relpath(source, SOURCE_DIR)
# build_args = build_args + " -Werror=duplicate-decl-specifier" # build_args = build_args + " -Werror=duplicate-decl-specifier"
with open(source, 'r', encoding='utf-8') as fh: with open(source, 'r', encoding='utf-8') as fh:
data = fh.read() data = fh.read()
@ -1416,7 +1435,7 @@ def wash_source_with_edit(
if skip_test: if skip_test:
# Just apply all edits. # Just apply all edits.
for (start, end), text, _text_always_fail, _extra_build_args in edits: for (start, end), text, _text_always_fail, _extra_build_args in edits:
data = apply_edit(data, text, start, end, verbose=VERBOSE) data = apply_edit(source_relative, data, text, start, end, verbose=verbose_edit_actions)
with open(source, 'w', encoding='utf-8') as fh: with open(source, 'w', encoding='utf-8') as fh:
fh.write(data) fh.write(data)
return return
@ -1424,6 +1443,9 @@ def wash_source_with_edit(
test_edit( test_edit(
source, output, None, build_args, build_cwd, data, data, source, output, None, build_args, build_cwd, data, data,
keep_edits=False, keep_edits=False,
expect_failure=False,
verbose_compile=verbose_compile,
verbose_edit_actions=verbose_edit_actions,
) )
if not os.path.exists(output): if not os.path.exists(output):
# raise Exception("Failed to produce output file: " + output) # raise Exception("Failed to produce output file: " + output)
@ -1448,18 +1470,24 @@ def wash_source_with_edit(
# Add directly after the compile command. # Add directly after the compile command.
build_args_for_edit = build_args[:1] + extra_build_args + build_args[1:] build_args_for_edit = build_args[:1] + extra_build_args + build_args[1:]
data_test = apply_edit(data, text, start, end, verbose=VERBOSE) data_test = apply_edit(source_relative, data, text, start, end, verbose=verbose_edit_actions)
if test_edit( if test_edit(
source, output, output_bytes, build_args_for_edit, build_cwd, data, data_test, source, output, output_bytes, build_args_for_edit, build_cwd, data, data_test,
keep_edits=False, keep_edits=False,
expect_failure=False,
verbose_compile=verbose_compile,
verbose_edit_actions=verbose_edit_actions,
): ):
# This worked, check if the change would fail if replaced with 'text_always_fail'. # This worked, check if the change would fail if replaced with 'text_always_fail'.
data_test_always_fail = apply_edit(data, text_always_fail, start, end, verbose=False) data_test_always_fail = apply_edit(source_relative, data, text_always_fail, start, end, verbose=False)
if test_edit( if test_edit(
source, output, output_bytes, build_args_for_edit, build_cwd, data, data_test_always_fail, source, output, output_bytes, build_args_for_edit, build_cwd, data, data_test_always_fail,
expect_failure=True, keep_edits=False, expect_failure=True,
keep_edits=False,
verbose_compile=verbose_compile,
verbose_edit_actions=verbose_edit_actions,
): ):
if VERBOSE_EDIT_ACTION: if verbose_edit_actions:
print("Edit at", (start, end), "doesn't fail, assumed to be ifdef'd out, continuing") print("Edit at", (start, end), "doesn't fail, assumed to be ifdef'd out, continuing")
continue continue
@ -1490,22 +1518,37 @@ def wash_source_with_edit_list(
build_args: Sequence[str], build_args: Sequence[str],
build_cwd: Optional[str], build_cwd: Optional[str],
skip_test: bool, skip_test: bool,
verbose_compile: bool,
verbose_edit_actions: bool,
shared_edit_data: Any, shared_edit_data: Any,
edit_list: Sequence[str], edit_list: Sequence[str],
) -> None: ) -> None:
for edit_to_apply in edit_list: for edit_to_apply in edit_list:
wash_source_with_edit(source, output, build_args, build_cwd, skip_test, shared_edit_data, edit_to_apply) wash_source_with_edit(
source,
output,
build_args,
build_cwd,
skip_test,
verbose_compile,
verbose_edit_actions,
shared_edit_data,
edit_to_apply,
)
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Edit Source Code From Args # Edit Source Code From Args
def run_edits_on_directory( def run_edits_on_directory(
*,
build_dir: str, build_dir: str,
regex_list: List[re.Pattern[str]], regex_list: List[re.Pattern[str]],
edits_to_apply: Sequence[str], edits_to_apply: Sequence[str],
skip_test: bool, skip_test: bool,
jobs: int, jobs: int,
verbose_compile: bool,
verbose_edit_actions: bool,
) -> int: ) -> int:
import multiprocessing import multiprocessing
@ -1606,7 +1649,7 @@ def run_edits_on_directory(
edits_to_apply_grouped = [[edit] for edit in edits_to_apply] edits_to_apply_grouped = [[edit] for edit in edits_to_apply]
for i, edits_group in enumerate(edits_to_apply_grouped): for i, edits_group in enumerate(edits_to_apply_grouped):
print("Applying edit:", edits_group, "({:d} of {:d})".format(i + 1, len(edits_to_apply_grouped))) print("Applying edit:", edits_group, "(%d of %d)" % (i + 1, len(edits_to_apply_grouped)))
edit_generator_class = edit_class_from_id(edits_group[0]) edit_generator_class = edit_class_from_id(edits_group[0])
shared_edit_data = edit_generator_class.setup() shared_edit_data = edit_generator_class.setup()
@ -1619,6 +1662,8 @@ def run_edits_on_directory(
build_args, build_args,
build_cwd, build_cwd,
skip_test, skip_test,
verbose_compile,
verbose_edit_actions,
shared_edit_data, shared_edit_data,
edits_group, edits_group,
) for (c, build_args, build_cwd) in args_with_cwd] ) for (c, build_args, build_cwd) in args_with_cwd]
@ -1634,6 +1679,8 @@ def run_edits_on_directory(
build_args, build_args,
build_cwd, build_cwd,
skip_test, skip_test,
verbose_compile,
verbose_edit_actions,
shared_edit_data, shared_edit_data,
edits_group, edits_group,
) )
@ -1649,7 +1696,7 @@ def run_edits_on_directory(
def create_parser(edits_all: Sequence[str]) -> argparse.ArgumentParser: def create_parser(edits_all: Sequence[str]) -> argparse.ArgumentParser:
from textwrap import indent from textwrap import indent
# Create docstring for edits. # Create doc-string for edits.
edits_all_docs = [] edits_all_docs = []
for edit in edits_all: for edit in edits_all:
# `%` -> `%%` is needed for `--help` not to interpret these as formatting arguments. # `%` -> `%%` is needed for `--help` not to interpret these as formatting arguments.
@ -1660,6 +1707,17 @@ def create_parser(edits_all: Sequence[str]) -> argparse.ArgumentParser:
) )
) )
# Create doc-string for verbose.
verbose_all_docs = []
for verbose_id, verbose_doc in VERBOSE_INFO:
# `%` -> `%%` is needed for `--help` not to interpret these as formatting arguments.
verbose_all_docs.append(
" %s\n%s" % (
verbose_id,
indent(verbose_doc.replace("%", "%%"), " "),
)
)
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description=__doc__, description=__doc__,
formatter_class=argparse.RawTextHelpFormatter, formatter_class=argparse.RawTextHelpFormatter,
@ -1684,6 +1742,16 @@ def create_parser(edits_all: Sequence[str]) -> argparse.ArgumentParser:
"Multiple edits may be passed at once (comma separated, no spaces)."), "Multiple edits may be passed at once (comma separated, no spaces)."),
required=True, required=True,
) )
parser.add_argument(
"--verbose",
dest="verbose",
default="",
help=(
"Specify verbose actions.\n\n" +
"\n".join(verbose_all_docs) + "\n"
"Multiple verbose types may be passed at once (comma separated, no spaces)."),
required=False,
)
parser.add_argument( parser.add_argument(
"--skip-test", "--skip-test",
dest="skip_test", dest="skip_test",
@ -1732,18 +1800,42 @@ def main() -> int:
for edit in edits_all_from_args: for edit in edits_all_from_args:
if edit not in edits_all: if edit not in edits_all:
print("Error, unrecognized '--edits' argument '{:s}', expected a value in {{{:s}}}".format( print("Error, unrecognized '--edits' argument '%s', expected a value in {%s}" % (
edit, edit,
", ".join(edits_all), ", ".join(edits_all),
)) ))
return 1 return 1
verbose_all = [verbose_id for verbose_id, _ in VERBOSE_INFO]
verbose_compile = False
verbose_edit_actions = False
verbose_all_from_args = args.verbose.split(",") if args.verbose else []
while verbose_all_from_args:
match (verbose_id := verbose_all_from_args.pop()):
case "compile":
verbose_compile = True
case "edit_actions":
verbose_edit_actions = True
case _:
print("Error, unrecognized '--verbose' argument '%s', expected a value in {%s}" % (
verbose_id,
", ".join(verbose_all),
))
return 1
if len(edits_all_from_args) > 1:
for edit in edits_all:
if edit not in edits_all_from_args:
print("Skipping edit:", edit)
return run_edits_on_directory( return run_edits_on_directory(
build_dir, build_dir=build_dir,
regex_list, regex_list=regex_list,
edits_all_from_args, edits_to_apply=edits_all_from_args,
args.skip_test, skip_test=args.skip_test,
args.jobs, jobs=args.jobs,
verbose_compile=verbose_compile,
verbose_edit_actions=verbose_edit_actions,
) )