1berry/cpython
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

gh-131423: Update OpenSSL data to 3.4.1 on Linux (#131618)

Browse Source 
OpenSSL 3.4.1 mnemonics are not compatible with OpenSSL 3.4.0 ones since
they were renumbered [1, 2]. Consequently, `_ssl_data_34.h` is renamed to
`_ssl_data_340.h` and `_ssl_data_34.h` now contains OpenSSL 3.4.1 mnemonics.

We also refine the mnemonics that are selected, discarding those that are
mnemonic-like but should not be used as such. More precisely, we remove
the ERR_LIB_MASK and ERR_LIB_OFFSET entries from OpenSSL 1.1.1 data.

[1]: https://github.com/openssl/openssl/issues/26316
[2]: https://github.com/openssl/openssl/issues/26388
This commit is contained in:
Bénédikt Tran 2025-04-25 10:26:58 +02:00 committed by GitHub
parent 9cba14881b
commit 6a9bfee4fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 9442 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/build.yml vendored
View File

@ -264,7 +264,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-24.04]
openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2, 3.4.0]
openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1]
# See Tools/ssl/make_ssl_data.py for notes on adding a new version
env:
OPENSSL_VER: ${{ matrix.openssl_ver }}
@ -331,7 +331,7 @@ jobs:
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
env:
OPENSSL_VER: 3.0.15
OPENSSL_VER: 3.0.16
PYTHONSTRICTEXTENSIONBUILD: 1
steps:
- uses: actions/checkout@v4
@ -450,7 +450,7 @@ jobs:
matrix:
os: [ubuntu-24.04]
env:
OPENSSL_VER: 3.0.15
OPENSSL_VER: 3.0.16
PYTHONSTRICTEXTENSIONBUILD: 1
ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
steps:

2
Doc/using/configure.rst
View File

@ -22,7 +22,7 @@ Features and minimum versions required to build CPython:
* Support for threads.
* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.9 is the recommended
* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.16 is the recommended
minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules.
* SQLite 3.15.2 for the :mod:`sqlite3` extension module.

2
Misc/NEWS.d/next/Library/2025-03-23-11-33-09.gh-issue-131423.bQlcEb.rst Normal file
View File

@ -0,0 +1,2 @@
:mod:`ssl` can show descriptions for errors added in OpenSSL 3.4.1.
Patch by Bénédikt Tran.

4
Modules/_ssl.c
View File

@ -121,8 +121,10 @@ static void _PySSLFixErrno(void) {
/* Include generated data (error codes) */
/* See make_ssl_data.h for notes on adding a new version. */
#if (OPENSSL_VERSION_NUMBER >= 0x30100000L)
#if (OPENSSL_VERSION_NUMBER >= 0x30401000L)
#include "_ssl_data_34.h"
#elif (OPENSSL_VERSION_NUMBER >= 0x30100000L)
#include "_ssl_data_340.h"
#elif (OPENSSL_VERSION_NUMBER >= 0x30000000L)
#include "_ssl_data_300.h"
#elif (OPENSSL_VERSION_NUMBER >= 0x10101000L)

11
Modules/_ssl_data_111.h
View File

@ -1,6 +1,8 @@
/* File generated by Tools/ssl/make_ssl_data.py */
/* Generated on 2024-11-27T12:48:46.194048+00:00 */
/* Generated on 2025-03-26T13:48:34.811613+00:00 */
/* Generated from Git commit OpenSSL_1_1_1w-0-ge04bd3433f */
/* generated from args.lib2errnum */
static struct py_ssl_library_code library_codes[] = {
#ifdef ERR_LIB_ASN1
{"ASN1", ERR_LIB_ASN1},
@ -137,10 +139,10 @@ static struct py_ssl_library_code library_codes[] = {
#ifdef ERR_LIB_X509V3
{"X509V3", ERR_LIB_X509V3},
#endif
{ NULL }
{NULL, 0} /* sentinel */
};
/* generated from args.reasons */
static struct py_ssl_error_code error_codes[] = {
#ifdef ASN1_R_ADDING_OBJECT
{"ADDING_OBJECT", ERR_LIB_ASN1, ASN1_R_ADDING_OBJECT},
@ -6537,6 +6539,5 @@ static struct py_ssl_error_code error_codes[] = {
#else
{"WRONG_TYPE", 11, 122},
#endif
{ NULL }
{NULL, 0, 0} /* sentinel */
};

46
Modules/_ssl_data_34.h
View File

@ -1,6 +1,8 @@
/* File generated by Tools/ssl/make_ssl_data.py */
/* Generated on 2024-11-27T12:35:52.276767+00:00 */
/* Generated from Git commit openssl-3.4.0-0-g98acb6b028 */
/* Generated on 2025-03-26T13:47:34.223146+00:00 */
/* Generated from Git commit openssl-3.4.1-0-ga26d85337d */
/* generated from args.lib2errnum */
static struct py_ssl_library_code library_codes[] = {
#ifdef ERR_LIB_ASN1
{"ASN1", ERR_LIB_ASN1},
@ -80,9 +82,6 @@ static struct py_ssl_library_code library_codes[] = {
#ifdef ERR_LIB_KDF
{"KDF", ERR_LIB_KDF},
#endif
#ifdef ERR_LIB_MASK
{"MASK", ERR_LIB_MASK},
#endif
#ifdef ERR_LIB_METH
{"METH", ERR_LIB_METH},
#endif
@ -95,9 +94,6 @@ static struct py_ssl_library_code library_codes[] = {
#ifdef ERR_LIB_OCSP
{"OCSP", ERR_LIB_OCSP},
#endif
#ifdef ERR_LIB_OFFSET
{"OFFSET", ERR_LIB_OFFSET},
#endif
#ifdef ERR_LIB_OSSL_DECODER
{"OSSL_DECODER", ERR_LIB_OSSL_DECODER},
#endif
@ -167,10 +163,10 @@ static struct py_ssl_library_code library_codes[] = {
#ifdef ERR_LIB_X509V3
{"X509V3", ERR_LIB_X509V3},
#endif
{ NULL }
{NULL, 0} /* sentinel */
};
/* generated from args.reasons */
static struct py_ssl_error_code error_codes[] = {
#ifdef ASN1_R_ADDING_OBJECT
{"ADDING_OBJECT", ERR_LIB_ASN1, ASN1_R_ADDING_OBJECT},
@ -1762,6 +1758,11 @@ static struct py_ssl_error_code error_codes[] = {
#else
{"ERROR_SETTING_RECIPIENTINFO", 46, 116},
#endif
#ifdef CMS_R_ERROR_UNSUPPORTED_STATIC_KEY_AGREEMENT
{"ERROR_UNSUPPORTED_STATIC_KEY_AGREEMENT", ERR_LIB_CMS, CMS_R_ERROR_UNSUPPORTED_STATIC_KEY_AGREEMENT},
#else
{"ERROR_UNSUPPORTED_STATIC_KEY_AGREEMENT", 46, 196},
#endif
#ifdef CMS_R_ESS_SIGNING_CERTID_MISMATCH_ERROR
{"ESS_SIGNING_CERTID_MISMATCH_ERROR", ERR_LIB_CMS, CMS_R_ESS_SIGNING_CERTID_MISMATCH_ERROR},
#else
@ -2122,25 +2123,15 @@ static struct py_ssl_error_code error_codes[] = {
#else
{"BROTLI_DECODE_ERROR", 41, 102},
#endif
#ifdef COMP_R_BROTLI_DEFLATE_ERROR
{"BROTLI_DEFLATE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_DEFLATE_ERROR},
#else
{"BROTLI_DEFLATE_ERROR", 41, 103},
#endif
#ifdef COMP_R_BROTLI_ENCODE_ERROR
{"BROTLI_ENCODE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_ENCODE_ERROR},
#else
{"BROTLI_ENCODE_ERROR", 41, 106},
#endif
#ifdef COMP_R_BROTLI_INFLATE_ERROR
{"BROTLI_INFLATE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_INFLATE_ERROR},
#else
{"BROTLI_INFLATE_ERROR", 41, 104},
{"BROTLI_ENCODE_ERROR", 41, 103},
#endif
#ifdef COMP_R_BROTLI_NOT_SUPPORTED
{"BROTLI_NOT_SUPPORTED", ERR_LIB_COMP, COMP_R_BROTLI_NOT_SUPPORTED},
#else
{"BROTLI_NOT_SUPPORTED", 41, 105},
{"BROTLI_NOT_SUPPORTED", 41, 104},
#endif
#ifdef COMP_R_ZLIB_DEFLATE_ERROR
{"ZLIB_DEFLATE_ERROR", ERR_LIB_COMP, COMP_R_ZLIB_DEFLATE_ERROR},
@ -2160,22 +2151,22 @@ static struct py_ssl_error_code error_codes[] = {
#ifdef COMP_R_ZSTD_COMPRESS_ERROR
{"ZSTD_COMPRESS_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_COMPRESS_ERROR},
#else
{"ZSTD_COMPRESS_ERROR", 41, 107},
{"ZSTD_COMPRESS_ERROR", 41, 105},
#endif
#ifdef COMP_R_ZSTD_DECODE_ERROR
{"ZSTD_DECODE_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_DECODE_ERROR},
#else
{"ZSTD_DECODE_ERROR", 41, 108},
{"ZSTD_DECODE_ERROR", 41, 106},
#endif
#ifdef COMP_R_ZSTD_DECOMPRESS_ERROR
{"ZSTD_DECOMPRESS_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_DECOMPRESS_ERROR},
#else
{"ZSTD_DECOMPRESS_ERROR", 41, 109},
{"ZSTD_DECOMPRESS_ERROR", 41, 107},
#endif
#ifdef COMP_R_ZSTD_NOT_SUPPORTED
{"ZSTD_NOT_SUPPORTED", ERR_LIB_COMP, COMP_R_ZSTD_NOT_SUPPORTED},
#else
{"ZSTD_NOT_SUPPORTED", 41, 110},
{"ZSTD_NOT_SUPPORTED", 41, 108},
#endif
#ifdef CONF_R_ERROR_LOADING_DSO
{"ERROR_LOADING_DSO", ERR_LIB_CONF, CONF_R_ERROR_LOADING_DSO},
@ -9272,6 +9263,5 @@ static struct py_ssl_error_code error_codes[] = {
#else
{"WRONG_TYPE", 11, 122},
#endif
{ NULL }
{NULL, 0, 0} /* sentinel */
};

9272
Modules/_ssl_data_340.h Normal file
View File

File diff suppressed because it is too large Load Diff

186
Tools/ssl/make_ssl_data.py
View File

@ -1,51 +1,100 @@
#! /usr/bin/env python3
#!/usr/bin/env python3
"""
This script should be called *manually* when we want to upgrade SSLError
`library` and `reason` mnemonics to a more recent OpenSSL version.
`library` and `reason` mnemonics to a more recent OpenSSL version. Note
that error codes are version specific.
It takes two arguments:
- the path to the OpenSSL git checkout
- the path to the header file to be generated Modules/_ssl_data_{version}.h
- error codes are version specific
- the path to the OpenSSL folder with the correct git checkout (see below)
- the path to the header file to be generated, usually
Modules/_ssl_data_<MAJOR><MINOR><PATCH>.h
The OpenSSL git checkout should be at a specific tag, using commands like:
git tag --list 'openssl-*'
git switch --detach openssl-3.4.0
git tag --list 'openssl-*'
git switch --detach openssl-3.4.1
After generating the definitions, compare the result with newest pre-existing file.
You can use a command like:
git diff --no-index Modules/_ssl_data_31.h Modules/_ssl_data_34.h
git diff --no-index Modules/_ssl_data_340.h Modules/_ssl_data_341.h
- If the new version *only* adds new definitions, remove the pre-existing file
and adjust the #include in _ssl.c to point to the new version.
- If the new version removes or renumbers some definitions, keep both files and
add a new #include in _ssl.c.
By convention, the latest OpenSSL mnemonics are gathered in the following file:
Modules/_ssl_data_<MAJOR><MINOR>.h
If those mnemonics are renumbered or removed in a subsequent OpenSSL version,
the file is renamed to "Modules/_ssl_data_<MAJOR><MINOR><PATCH>.h" and the
latest mnemonics are stored in the patchless file (see below for an example).
A newly supported OpenSSL version should also be added to:
- Tools/ssl/multissltests.py
- .github/workflows/build.yml
Example: new mnemonics are added
--------------------------------
Assume that "Modules/_ssl_data_32x.h" contains the latest mnemonics for
CPython and was generated from OpenSSL 3.2.1. If only new mnemonics are
added in OpenSSL 3.2.2, the following commands should be executed:
# in the OpenSSL git directory
git switch --detach openssl-3.2.2
# in the CPython git directory
python make_ssl_data.py PATH_TO_OPENSSL_GIT_CLONE Modules/_ssl_data_322.h
mv Modules/_ssl_data_322.h Modules/_ssl_data_32.h
Example: mnemonics are renamed/removed
--------------------------------------
Assume that the existing file is Modules/_ssl_data_34x.h and is based
on OpenSSL 3.4.0. Since some mnemonics were renamed in OpenSSL 3.4.1,
the following commands should be executed:
# in the OpenSSL git directory
git switch --detach openssl-3.4.1
# in the CPython git directory
mv Modules/_ssl_data_34.h Modules/_ssl_data_340.h
python make_ssl_data.py PATH_TO_OPENSSL_GIT_CLONE Modules/_ssl_data_341.h
mv Modules/_ssl_data_341.h Modules/_ssl_data_34.h
"""
import argparse
import datetime
import logging
import operator
import os
import re
import subprocess
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser(
description="Generate ssl_data.h from OpenSSL sources"
formatter_class=argparse.RawTextHelpFormatter,
description="Generate SSL data headers from OpenSSL sources"
)
parser.add_argument("srcdir", help="OpenSSL source directory")
parser.add_argument(
"output", nargs="?", default=None
"output", nargs="?", default=None,
help="output file (default: standard output)",
)
def error(format_string, *format_args, **kwargs):
# do not use parser.error() to avoid printing short help
logger.error(format_string, *format_args, **kwargs)
raise SystemExit(1)
def _file_search(fname, pat):
with open(fname, encoding="utf-8") as f:
for line in f:
@ -55,52 +104,73 @@ def _file_search(fname, pat):
def parse_err_h(args):
"""Parse err codes, e.g. ERR_LIB_X509: 11"""
pat = re.compile(r"#\s*define\W+ERR_LIB_(\w+)\s+(\d+)")
"""Parse error codes from include/openssl/err.h.in.
Detected lines match (up to spaces) "#define ERR_LIB_<LIBNAME> <ERRCODE>",
e.g., "# define ERR_LIB_NONE 1".
"""
pat = re.compile(r"#\s*define\W+(ERR_LIB_(\w+))\s+(\d+)")
lib2errnum = {}
for match in _file_search(args.err_h, pat):
libname, num = match.groups()
lib2errnum[libname] = int(num)
macroname, libname, num = match.groups()
if macroname in ['ERR_LIB_OFFSET', 'ERR_LIB_MASK']:
# ignore: "# define ERR_LIB_OFFSET 23L"
# ignore: "# define ERR_LIB_MASK 0xFF"
continue
actual = int(num)
expect = lib2errnum.setdefault(libname, actual)
if actual != expect:
logger.warning("OpenSSL inconsistency for ERR_LIB_%s (%d != %d)",
libname, actual, expect)
return lib2errnum
def parse_openssl_error_text(args):
"""Parse error reasons, X509_R_AKID_MISMATCH"""
# ignore backslash line continuation for now
"""Parse error reasons from crypto/err/openssl.txt.
Detected lines match "<LIBNAME>_R_<ERRNAME>:<ERRCODE>:<MESSAGE>",
e.g., "ASN1_R_ADDING_OBJECT:171:adding object". The <MESSAGE> part
is not stored as it will be recovered at runtime when needed.
"""
# ignore backslash line continuation (placed before <MESSAGE> if present)
pat = re.compile(r"^((\w+?)_R_(\w+)):(\d+):")
seen = {}
for match in _file_search(args.errtxt, pat):
reason, libname, errname, num = match.groups()
if "_F_" in reason:
# ignore function codes
if "_F_" in reason: # ignore function codes
# FEAT(picnixz): in the future, we may want to also check
# the consistency of the OpenSSL files with an external tool.
continue
num = int(num)
yield reason, libname, errname, num
yield reason, libname, errname, int(num)
def parse_extra_reasons(args):
"""Parse extra reasons from openssl.ec"""
"""Parse extra reasons from crypto/err/openssl.ec.
Detected lines are matched against "R <LIBNAME>_R_<ERRNAME> <ERRCODE>",
e.g., "R SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE 1010".
"""
pat = re.compile(r"^R\s+((\w+)_R_(\w+))\s+(\d+)")
for match in _file_search(args.errcodes, pat):
reason, libname, errname, num = match.groups()
num = int(num)
yield reason, libname, errname, num
yield reason, libname, errname, int(num)
def gen_library_codes(args):
"""Generate table short libname to numeric code"""
"""Generate table short libname to numeric code."""
yield "/* generated from args.lib2errnum */"
yield "static struct py_ssl_library_code library_codes[] = {"
for libname in sorted(args.lib2errnum):
yield f"#ifdef ERR_LIB_{libname}"
yield f' {{"{libname}", ERR_LIB_{libname}}},'
yield "#endif"
yield " { NULL }"
yield " {NULL, 0} /* sentinel */"
yield "};"
yield ""
def gen_error_codes(args):
"""Generate error code table for error reasons"""
"""Generate error code table for error reasons."""
yield "/* generated from args.reasons */"
yield "static struct py_ssl_error_code error_codes[] = {"
for reason, libname, errname, num in args.reasons:
yield f" #ifdef {reason}"
@ -108,35 +178,11 @@ def gen_error_codes(args):
yield " #else"
yield f' {{"{errname}", {args.lib2errnum[libname]}, {num}}},'
yield " #endif"
yield " { NULL }"
yield " {NULL, 0, 0} /* sentinel */"
yield "};"
yield ""
def main():
args = parser.parse_args()
args.err_h = os.path.join(args.srcdir, "include", "openssl", "err.h")
if not os.path.isfile(args.err_h):
# Fall back to infile for OpenSSL 3.0.0
args.err_h += ".in"
args.errcodes = os.path.join(args.srcdir, "crypto", "err", "openssl.ec")
args.errtxt = os.path.join(args.srcdir, "crypto", "err", "openssl.txt")
if not os.path.isfile(args.errtxt):
parser.error(f"File {args.errtxt} not found in srcdir\n.")
# {X509: 11, ...}
args.lib2errnum = parse_err_h(args)
# [('X509_R_AKID_MISMATCH', 'X509', 'AKID_MISMATCH', 110), ...]
reasons = []
reasons.extend(parse_openssl_error_text(args))
reasons.extend(parse_extra_reasons(args))
# sort by libname, numeric error code
args.reasons = sorted(reasons, key=operator.itemgetter(0, 3))
def get_openssl_git_commit(args):
git_describe = subprocess.run(
['git', 'describe', '--long', '--dirty'],
cwd=args.srcdir,
@ -144,10 +190,40 @@ def main():
encoding='utf-8',
check=True,
)
return git_describe.stdout.strip()
def main(args=None):
args = parser.parse_args(args)
if not os.path.isdir(args.srcdir):
error(f"OpenSSL directory not found: {args.srcdir}")
args.err_h = os.path.join(args.srcdir, "include", "openssl", "err.h")
if not os.path.isfile(args.err_h):
# Fall back to infile for OpenSSL 3.0.0 and later.
args.err_h += ".in"
args.errcodes = os.path.join(args.srcdir, "crypto", "err", "openssl.ec")
if not os.path.isfile(args.errcodes):
error(f"file {args.errcodes} not found in {args.srcdir}")
args.errtxt = os.path.join(args.srcdir, "crypto", "err", "openssl.txt")
if not os.path.isfile(args.errtxt):
error(f"file {args.errtxt} not found in {args.srcdir}")
# [("ERR_LIB_X509", "X509", 11), ...]
args.lib2errnum = parse_err_h(args)
# [('X509_R_AKID_MISMATCH', 'X509', 'AKID_MISMATCH', 110), ...]
reasons = []
reasons.extend(parse_openssl_error_text(args))
reasons.extend(parse_extra_reasons(args))
# sort by macro name and numeric error code
args.reasons = sorted(reasons, key=operator.itemgetter(0, 3))
commit = get_openssl_git_commit(args)
lines = [
"/* File generated by Tools/ssl/make_ssl_data.py */",
f"/* Generated on {datetime.datetime.now(datetime.UTC).isoformat()} */",
f"/* Generated from Git commit {git_describe.stdout.strip()} */",
f"/* Generated from Git commit {commit} */",
"",
]
lines.extend(gen_library_codes(args))
lines.append("")

10
Tools/ssl/multissltests.py
View File

@ -47,11 +47,11 @@ OPENSSL_OLD_VERSIONS = [
]
OPENSSL_RECENT_VERSIONS = [
"3.0.15",
"3.1.7",
"3.2.3",
"3.3.2",
"3.4.0",
"3.0.16",
"3.1.8",
"3.2.4",
"3.3.3",
"3.4.1",
# See make_ssl_data.py for notes on adding a new version.
]