gh-131531: android.py enhancements to support cibuildwheel (#132870)

Modifies the environment handling and execution arguments of the Android management
script to support the compilation of third-party binaries, and the use of the testbed to 
invoke third-party test code.

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-authored-by: Russell Keith-Magee <russell@keith-magee.com>

Android/README.md

@@ -156,6 +156,10 @@ repository's `Lib` directory will be picked up immediately. Changes in C files,
 and architecture-specific files such as sysconfigdata, will not take effect
 until you re-run `android.py make-host` or `build`.
 
+The testbed app can also be used to test third-party packages. For more details,
+run `android.py test --help`, paying attention to the options `--site-packages`,
+`--cwd`, `-c` and `-m`.
+
 
 ## Using in your own app
 

Android/android-env.sh

@@ -3,7 +3,7 @@
 : "${HOST:?}"  # GNU target triplet
 
 # You may also override the following:
-: "${api_level:=24}"  # Minimum Android API level the build will run on
+: "${ANDROID_API_LEVEL:=24}"  # Minimum Android API level the build will run on
 : "${PREFIX:-}"  # Path in which to find required libraries
 
 
@@ -24,7 +24,7 @@ fail() {
 # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md
 #   where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.:
 #   https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md
-ndk_version=27.1.12297006
+ndk_version=27.2.12479018
 
 ndk=$ANDROID_HOME/ndk/$ndk_version
 if ! [ -e "$ndk" ]; then
@@ -43,7 +43,7 @@ fi
 toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*)
 export AR="$toolchain/bin/llvm-ar"
 export AS="$toolchain/bin/llvm-as"
-export CC="$toolchain/bin/${clang_triplet}${api_level}-clang"
+export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang"
 export CXX="${CC}++"
 export LD="$toolchain/bin/ld"
 export NM="$toolchain/bin/llvm-nm"

Android/android.py

@@ -14,7 +14,7 @@ from asyncio import wait_for
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from glob import glob
-from os.path import basename, relpath
+from os.path import abspath, basename, relpath
 from pathlib import Path
 from subprocess import CalledProcessError
 from tempfile import TemporaryDirectory
@@ -22,9 +22,13 @@ from tempfile import TemporaryDirectory
 
 SCRIPT_NAME = Path(__file__).name
 ANDROID_DIR = Path(__file__).resolve().parent
-CHECKOUT = ANDROID_DIR.parent
+PYTHON_DIR = ANDROID_DIR.parent
+in_source_tree = (
+    ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists()
+)
+
 TESTBED_DIR = ANDROID_DIR / "testbed"
-CROSS_BUILD_DIR = CHECKOUT / "cross-build"
+CROSS_BUILD_DIR = PYTHON_DIR / "cross-build"
 
 HOSTS = ["aarch64-linux-android", "x86_64-linux-android"]
 APP_ID = "org.python.testbed"
@@ -76,39 +80,68 @@ def run(command, *, host=None, env=None, log=True, **kwargs):
     kwargs.setdefault("check", True)
     if env is None:
         env = os.environ.copy()
-    original_env = env.copy()
 
     if host:
-        env_script = ANDROID_DIR / "android-env.sh"
+        host_env = android_env(host)
-        env_output = subprocess.run(
+        print_env(host_env)
-            f"set -eu; "
+        env.update(host_env)
-            f"HOST={host}; "
-            f"PREFIX={subdir(host)}/prefix; "
-            f". {env_script}; "
-            f"export",
-            check=True, shell=True, text=True, stdout=subprocess.PIPE
-        ).stdout
-
-        for line in env_output.splitlines():
-            # We don't require every line to match, as there may be some other
-            # output from installing the NDK.
-            if match := re.search(
-                "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line
-            ):
-                key, value = match[2], match[3]
-                if env.get(key) != value:
-                    print(line)
-                    env[key] = value
-
-        if env == original_env:
-            raise ValueError(f"Found no variables in {env_script.name} output:\n"
-                             + env_output)
 
     if log:
-        print(">", " ".join(map(str, command)))
+        print(">", join_command(command))
     return subprocess.run(command, env=env, **kwargs)
 
 
+# Format a command so it can be copied into a shell. Like shlex.join, but also
+# accepts arguments which are Paths, or a single string/Path outside of a list.
+def join_command(args):
+    if isinstance(args, (str, Path)):
+        return str(args)
+    else:
+        return shlex.join(map(str, args))
+
+
+# Format the environment so it can be pasted into a shell.
+def print_env(env):
+    for key, value in sorted(env.items()):
+        print(f"export {key}={shlex.quote(value)}")
+
+
+def android_env(host):
+    if host:
+        prefix = subdir(host) / "prefix"
+    else:
+        prefix = ANDROID_DIR / "prefix"
+        sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py")
+        sysconfig_filename = next(sysconfig_files).name
+        host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1]
+
+    env_script = ANDROID_DIR / "android-env.sh"
+    env_output = subprocess.run(
+        f"set -eu; "
+        f"export HOST={host}; "
+        f"PREFIX={prefix}; "
+        f". {env_script}; "
+        f"export",
+        check=True, shell=True, capture_output=True, encoding='utf-8',
+    ).stdout
+
+    env = {}
+    for line in env_output.splitlines():
+        # We don't require every line to match, as there may be some other
+        # output from installing the NDK.
+        if match := re.search(
+            "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line
+        ):
+            key, value = match[2], match[3]
+            if os.environ.get(key) != value:
+                env[key] = value
+
+    if not env:
+        raise ValueError(f"Found no variables in {env_script.name} output:\n"
+                         + env_output)
+    return env
+
+
 def build_python_path():
     """The path to the build Python binary."""
     build_dir = subdir("build")
@@ -127,7 +160,7 @@ def configure_build_python(context):
         clean("build")
     os.chdir(subdir("build", create=True))
 
-    command = [relpath(CHECKOUT / "configure")]
+    command = [relpath(PYTHON_DIR / "configure")]
     if context.args:
         command.extend(context.args)
     run(command)
@@ -139,12 +172,13 @@ def make_build_python(context):
 
 
 def unpack_deps(host, prefix_dir):
+    os.chdir(prefix_dir)
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
-    for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4",
+    for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4",
                      "sqlite-3.49.1-0", "xz-5.4.6-1"]:
         filename = f"{name_ver}-{host}.tar.gz"
         download(f"{deps_url}/{name_ver}/{filename}")
-        shutil.unpack_archive(filename, prefix_dir)
+        shutil.unpack_archive(filename)
         os.remove(filename)
 
 
@@ -167,7 +201,7 @@ def configure_host_python(context):
     os.chdir(host_dir)
     command = [
         # Basic cross-compiling configuration
-        relpath(CHECKOUT / "configure"),
+        relpath(PYTHON_DIR / "configure"),
         f"--host={context.host}",
         f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}",
         f"--with-build-python={build_python_path()}",
@@ -196,9 +230,12 @@ def make_host_python(context):
     for pattern in ("include/python*", "lib/libpython*", "lib/python*"):
         delete_glob(f"{prefix_dir}/{pattern}")
 
+    # The Android environment variables were already captured in the Makefile by
+    # `configure`, and passing them again when running `make` may cause some
+    # flags to be duplicated. So we don't use the `host` argument here.
     os.chdir(host_dir)
-    run(["make", "-j", str(os.cpu_count())], host=context.host)
+    run(["make", "-j", str(os.cpu_count())])
-    run(["make", "install", f"prefix={prefix_dir}"], host=context.host)
+    run(["make", "install", f"prefix={prefix_dir}"])
 
 
 def build_all(context):
@@ -228,7 +265,12 @@ def setup_sdk():
     if not all((android_home / "licenses" / path).exists() for path in [
         "android-sdk-arm-dbt-license", "android-sdk-license"
     ]):
-        run([sdkmanager, "--licenses"], text=True, input="y\n" * 100)
+        run(
+            [sdkmanager, "--licenses"],
+            text=True,
+            capture_output=True,
+            input="y\n" * 100,
+        )
 
     # Gradle may install this automatically, but we can't rely on that because
     # we need to run adb within the logcat task.
@@ -474,24 +516,49 @@ async def gradle_task(context):
         task_prefix = "connected"
         env["ANDROID_SERIAL"] = context.connected
 
+    hidden_output = []
+
+    def log(line):
+        # Gradle may take several minutes to install SDK packages, so it's worth
+        # showing those messages even in non-verbose mode.
+        if context.verbose or line.startswith('Preparing "Install'):
+            sys.stdout.write(line)
+        else:
+            hidden_output.append(line)
+
+    if context.command:
+        mode = "-c"
+        module = context.command
+    else:
+        mode = "-m"
+        module = context.module or "test"
+
     args = [
         gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest",
-        "-Pandroid.testInstrumentationRunnerArguments.pythonArgs="
+    ] + [
-        + shlex.join(context.args),
+        # Build-time properties
+        f"-Ppython.{name}={value}"
+        for name, value in [
+            ("sitePackages", context.site_packages), ("cwd", context.cwd)
+        ] if value
+    ] + [
+        # Runtime properties
+        f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}"
+        for name, value in [
+            ("Mode", mode), ("Module", module), ("Args", join_command(context.args))
+        ] if value
     ]
-    hidden_output = []
+    if context.verbose >= 2:
+        args.append("--info")
+    log("> " + join_command(args))
+
     try:
         async with async_process(
             *args, cwd=TESTBED_DIR, env=env,
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
         ) as process:
             while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
-                # Gradle may take several minutes to install SDK packages, so
+                log(line)
-                # it's worth showing those messages even in non-verbose mode.
-                if context.verbose or line.startswith('Preparing "Install'):
-                    sys.stdout.write(line)
-                else:
-                    hidden_output.append(line)
 
             status = await wait_for(process.wait(), timeout=1)
             if status == 0:
@@ -604,6 +671,10 @@ def package(context):
         print(f"Wrote {package_path}")
 
 
+def env(context):
+    print_env(android_env(getattr(context, "host", None)))
+
+
 # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated
 # by the buildbot worker, we'll make an attempt to clean up our subprocesses.
 def install_signal_handler():
@@ -615,36 +686,41 @@ def install_signal_handler():
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    subcommands = parser.add_subparsers(dest="subcommand")
+    subcommands = parser.add_subparsers(dest="subcommand", required=True)
 
     # Subcommands
-    build = subcommands.add_parser("build", help="Build everything")
+    build = subcommands.add_parser(
-    configure_build = subcommands.add_parser("configure-build",
+        "build", help="Run configure-build, make-build, configure-host and "
-                                             help="Run `configure` for the "
+        "make-host")
-                                             "build Python")
+    configure_build = subcommands.add_parser(
-    make_build = subcommands.add_parser("make-build",
+        "configure-build", help="Run `configure` for the build Python")
-                                        help="Run `make` for the build Python")
-    configure_host = subcommands.add_parser("configure-host",
-                                            help="Run `configure` for Android")
-    make_host = subcommands.add_parser("make-host",
-                                       help="Run `make` for Android")
     subcommands.add_parser(
-        "clean", help="Delete all build and prefix directories")
+        "make-build", help="Run `make` for the build Python")
-    subcommands.add_parser(
+    configure_host = subcommands.add_parser(
-        "build-testbed", help="Build the testbed app")
+        "configure-host", help="Run `configure` for Android")
-    test = subcommands.add_parser(
+    make_host = subcommands.add_parser(
-        "test", help="Run the test suite")
+        "make-host", help="Run `make` for Android")
+
+    subcommands.add_parser("clean", help="Delete all build directories")
+    subcommands.add_parser("build-testbed", help="Build the testbed app")
+    test = subcommands.add_parser("test", help="Run the testbed app")
     package = subcommands.add_parser("package", help="Make a release package")
+    env = subcommands.add_parser("env", help="Print environment variables")
 
     # Common arguments
     for subcommand in build, configure_build, configure_host:
         subcommand.add_argument(
             "--clean", action="store_true", default=False, dest="clean",
-            help="Delete the relevant build and prefix directories first")
+            help="Delete the relevant build directories first")
-    for subcommand in [build, configure_host, make_host, package]:
+
+    host_commands = [build, configure_host, make_host, package]
+    if in_source_tree:
+        host_commands.append(env)
+    for subcommand in host_commands:
         subcommand.add_argument(
             "host", metavar="HOST", choices=HOSTS,
             help="Host triplet: choices=[%(choices)s]")
+
     for subcommand in build, configure_build, configure_host:
         subcommand.add_argument("args", nargs="*",
                                 help="Extra arguments to pass to `configure`")
@@ -654,6 +730,7 @@ def parse_args():
         "-v", "--verbose", action="count", default=0,
         help="Show Gradle output, and non-Python logcat messages. "
         "Use twice to include high-volume messages which are rarely useful.")
+
     device_group = test.add_mutually_exclusive_group(required=True)
     device_group.add_argument(
         "--connected", metavar="SERIAL", help="Run on a connected device. "
@@ -661,8 +738,24 @@ def parse_args():
     device_group.add_argument(
         "--managed", metavar="NAME", help="Run on a Gradle-managed device. "
         "These are defined in `managedDevices` in testbed/app/build.gradle.kts.")
+
     test.add_argument(
-        "args", nargs="*", help=f"Arguments for `python -m test`. "
+        "--site-packages", metavar="DIR", type=abspath,
+        help="Directory to copy as the app's site-packages.")
+    test.add_argument(
+        "--cwd", metavar="DIR", type=abspath,
+        help="Directory to copy as the app's working directory.")
+
+    mode_group = test.add_mutually_exclusive_group()
+    mode_group.add_argument(
+        "-c", dest="command", help="Execute the given Python code.")
+    mode_group.add_argument(
+        "-m", dest="module", help="Execute the module with the given name.")
+    test.epilog = (
+        "If neither -c nor -m are passed, the default is '-m test', which will "
+        "run Python's own test suite.")
+    test.add_argument(
+        "args", nargs="*", help=f"Arguments to add to sys.argv. "
         f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.")
 
     return parser.parse_args()
@@ -688,6 +781,7 @@ def main():
         "build-testbed": build_testbed,
         "test": run_testbed,
         "package": package,
+        "env": env,
     }
 
     try:
@@ -708,14 +802,9 @@ def print_called_process_error(e):
             if not content.endswith("\n"):
                 stream.write("\n")
 
-    # Format the command so it can be copied into a shell. shlex uses single
+    # shlex uses single quotes, so we surround the command with double quotes.
-    # quotes, so we surround the whole command with double quotes.
-    args_joined = (
-        e.cmd if isinstance(e.cmd, str)
-        else " ".join(shlex.quote(str(arg)) for arg in e.cmd)
-    )
     print(
-        f'Command "{args_joined}" returned exit status {e.returncode}'
+        f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}'
     )
 
 

Android/testbed/app/build.gradle.kts

@@ -85,7 +85,7 @@ android {
 
         minSdk = androidEnvFile.useLines {
             for (line in it) {
-                """api_level:=(\d+)""".toRegex().find(line)?.let {
+                """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let {
                     return@useLines it.groupValues[1].toInt()
                 }
             }
@@ -205,11 +205,29 @@ androidComponents.onVariants { variant ->
 
                 into("site-packages") {
                     from("$projectDir/src/main/python")
+
+                    val sitePackages = findProperty("python.sitePackages") as String?
+                    if (!sitePackages.isNullOrEmpty()) {
+                        if (!file(sitePackages).exists()) {
+                            throw GradleException("$sitePackages does not exist")
+                        }
+                        from(sitePackages)
+                    }
                 }
 
                 duplicatesStrategy = DuplicatesStrategy.EXCLUDE
                 exclude("**/__pycache__")
             }
+
+            into("cwd") {
+                val cwd = findProperty("python.cwd") as String?
+                if (!cwd.isNullOrEmpty()) {
+                    if (!file(cwd).exists()) {
+                        throw GradleException("$cwd does not exist")
+                    }
+                    from(cwd)
+                }
+            }
         }
     }
 

Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt

@@ -17,11 +17,11 @@ class PythonSuite {
     fun testPython() {
         val start = System.currentTimeMillis()
         try {
-            val context =
+            val status = PythonTestRunner(
                 InstrumentationRegistry.getInstrumentation().targetContext
-            val args =
+            ).run(
-                InstrumentationRegistry.getArguments().getString("pythonArgs", "")
+                InstrumentationRegistry.getArguments()
-            val status = PythonTestRunner(context).run(args)
+            )
             assertEquals(0, status)
         } finally {
             // Make sure the process lives long enough for the test script to

Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt

@@ -15,17 +15,29 @@ class MainActivity : AppCompatActivity() {
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         setContentView(R.layout.activity_main)
-        val status = PythonTestRunner(this).run("-W -uall")
+        val status = PythonTestRunner(this).run("-m", "test", "-W -uall")
         findViewById<TextView>(R.id.tvHello).text = "Exit status $status"
     }
 }
 
 
 class PythonTestRunner(val context: Context) {
-    /** @param args Extra arguments for `python -m test`.
+    fun run(instrumentationArgs: Bundle) = run(
-     * @return The Python exit status: zero if the tests passed, nonzero if
+        instrumentationArgs.getString("pythonMode")!!,
-     * they failed. */
+        instrumentationArgs.getString("pythonModule")!!,
-    fun run(args: String = "") : Int {
+        instrumentationArgs.getString("pythonArgs") ?: "",
+    )
+
+    /** Run Python.
+     *
+     * @param mode Either "-c" or "-m".
+     * @param module Python statements for "-c" mode, or a module name for
+     *     "-m" mode.
+     * @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`.
+     * @return The Python exit status: zero on success, nonzero on failure. */
+    fun run(mode: String, module: String, args: String) : Int {
+        Os.setenv("PYTHON_MODE", mode, true)
+        Os.setenv("PYTHON_MODULE", module, true)
         Os.setenv("PYTHON_ARGS", args, true)
 
         // Python needs this variable to help it find the temporary directory,
@@ -36,8 +48,9 @@ class PythonTestRunner(val context: Context) {
         System.loadLibrary("main_activity")
         redirectStdioToLogcat()
 
-        // The main module is in src/main/python/main.py.
+        // The main module is in src/main/python. We don't simply call it
-        return runPython(pythonHome.toString(), "main")
+        // "main", as that could clash with third-party test code.
+        return runPython(pythonHome.toString(), "android_testbed_main")
     }
 
     private fun extractAssets() : File {

Android/testbed/app/src/main/python/android_testbed_main.py

@@ -26,7 +26,23 @@ import sys
 #     test_signals in test_threadsignals.py.
 signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1])
 
+mode = os.environ["PYTHON_MODE"]
+module = os.environ["PYTHON_MODULE"]
 sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"])
 
-# The test module will call sys.exit to indicate whether the tests passed.
+cwd = f"{sys.prefix}/cwd"
-runpy.run_module("test")
+if not os.path.exists(cwd):
+    # Empty directories are lost in the asset packing/unpacking process.
+    os.mkdir(cwd)
+os.chdir(cwd)
+
+if mode == "-c":
+    # In -c mode, sys.path starts with an empty string, which means whatever the current
+    # working directory is at the moment of each import.
+    sys.path.insert(0, "")
+    exec(module, {})
+elif mode == "-m":
+    sys.path.insert(0, os.getcwd())
+    runpy.run_module(module, run_name="__main__", alter_sys=True)
+else:
+    raise ValueError(f"unknown mode: {mode}")

Android/testbed/build.gradle.kts

@@ -1,5 +1,5 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 plugins {
-    id("com.android.application") version "8.6.1" apply false
+    id("com.android.application") version "8.10.0" apply false
     id("org.jetbrains.kotlin.android") version "1.9.22" apply false
 }

Android/testbed/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 #Mon Feb 19 20:29:06 GMT 2024
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

Doc/using/android.rst

@@ -63,3 +63,12 @@ link to the relevant file.
 * Add code to your app to :source:`start Python in embedded mode
   <Android/testbed/app/src/main/c/main_activity.c>`. This will need to be C code
   called via JNI.
+
+Building a Python package for Android
+-------------------------------------
+
+Python packages can be built for Android as wheels and released on PyPI. The
+recommended tool for doing this is `cibuildwheel
+<https://cibuildwheel.pypa.io/en/stable/platforms/#android>`__, which automates
+all the details of setting up a cross-compilation environment, building the
+wheel, and testing it on an emulator.