gh-99352: Respect http.client.HTTPConnection.debuglevel in urllib.request.AbstractHTTPHandler (#99353)

* bugfix: let the HTTP- and HTTPSHandlers respect the value of http.client.HTTPConnection.debuglevel

* add tests

* add news

* ReSTify NEWS and reword a bit.

* Address Review Comments.

* Use mock.patch.object instead of settting the module level value.
* Used test values to assert the debuglevel.

---------

Co-authored-by: Gregory P. Smith <greg@krypto.org>
Co-authored-by: Senthil Kumaran <senthil@python.org>

Lib/test/test_urllib2.py

@@ -3,6 +3,7 @@ from test import support
 from test.support import os_helper
 from test.support import warnings_helper
 from test import test_urllib
+from unittest import mock
 
 import os
 import io
@@ -484,7 +485,18 @@ def build_test_opener(*handler_instances):
     return opener
 
 
-class MockHTTPHandler(urllib.request.BaseHandler):
+class MockHTTPHandler(urllib.request.HTTPHandler):
+    # Very simple mock HTTP handler with no special behavior other than using a mock HTTP connection
+
+    def __init__(self, debuglevel=None):
+        super(MockHTTPHandler, self).__init__(debuglevel=debuglevel)
+        self.httpconn = MockHTTPClass()
+
+    def http_open(self, req):
+        return self.do_open(self.httpconn, req)
+
+
+class MockHTTPHandlerRedirect(urllib.request.BaseHandler):
     # useful for testing redirections and auth
     # sends supplied headers and code as first response
     # sends 200 OK as second response
@@ -512,12 +524,12 @@ class MockHTTPHandler(urllib.request.BaseHandler):
             return MockResponse(200, "OK", msg, "", req.get_full_url())
 
 
-class MockHTTPSHandler(urllib.request.AbstractHTTPHandler):
+class MockHTTPSHandler(urllib.request.HTTPSHandler):
     # Useful for testing the Proxy-Authorization request by verifying the
     # properties of httpcon
 
-    def __init__(self, debuglevel=0):
+    def __init__(self, debuglevel=None, context=None, check_hostname=None):
-        urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel)
+        super(MockHTTPSHandler, self).__init__(debuglevel, context, check_hostname)
         self.httpconn = MockHTTPClass()
 
     def https_open(self, req):
@@ -1048,12 +1060,35 @@ class HandlerTests(unittest.TestCase):
             newreq = h.do_request_(req)
             self.assertEqual(int(newreq.get_header('Content-length')),16)
 
-    def test_http_handler_debuglevel(self):
+    def test_http_handler_global_debuglevel(self):
+        with mock.patch.object(http.client.HTTPConnection, 'debuglevel', 6):
+            o = OpenerDirector()
+            h = MockHTTPHandler()
+            o.add_handler(h)
+            o.open("http://www.example.com")
+            self.assertEqual(h._debuglevel, 6)
+
+    def test_http_handler_local_debuglevel(self):
         o = OpenerDirector()
-        h = MockHTTPSHandler(debuglevel=1)
+        h = MockHTTPHandler(debuglevel=5)
+        o.add_handler(h)
+        o.open("http://www.example.com")
+        self.assertEqual(h._debuglevel, 5)
+
+    def test_https_handler_global_debuglevel(self):
+        with mock.patch.object(http.client.HTTPSConnection, 'debuglevel', 7):
+            o = OpenerDirector()
+            h = MockHTTPSHandler()
+            o.add_handler(h)
+            o.open("https://www.example.com")
+            self.assertEqual(h._debuglevel, 7)
+
+    def test_https_handler_local_debuglevel(self):
+        o = OpenerDirector()
+        h = MockHTTPSHandler(debuglevel=4)
         o.add_handler(h)
         o.open("https://www.example.com")
-        self.assertEqual(h._debuglevel, 1)
+        self.assertEqual(h._debuglevel, 4)
 
     def test_http_doubleslash(self):
         # Checks the presence of any unnecessary double slash in url does not
@@ -1289,7 +1324,7 @@ class HandlerTests(unittest.TestCase):
 
         cj = CookieJar()
         interact_netscape(cj, "http://www.example.com/", "spam=eggs")
-        hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n")
+        hh = MockHTTPHandlerRedirect(302, "Location: http://www.cracker.com/\r\n\r\n")
         hdeh = urllib.request.HTTPDefaultErrorHandler()
         hrh = urllib.request.HTTPRedirectHandler()
         cp = urllib.request.HTTPCookieProcessor(cj)
@@ -1299,7 +1334,7 @@ class HandlerTests(unittest.TestCase):
 
     def test_redirect_fragment(self):
         redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n'
-        hh = MockHTTPHandler(302, 'Location: ' + redirected_url)
+        hh = MockHTTPHandlerRedirect(302, 'Location: ' + redirected_url)
         hdeh = urllib.request.HTTPDefaultErrorHandler()
         hrh = urllib.request.HTTPRedirectHandler()
         o = build_test_opener(hh, hdeh, hrh)
@@ -1484,7 +1519,7 @@ class HandlerTests(unittest.TestCase):
             password_manager = MockPasswordManager()
             auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
             body = '\r\n'.join(headers) + '\r\n\r\n'
-            http_handler = MockHTTPHandler(401, body)
+            http_handler = MockHTTPHandlerRedirect(401, body)
             opener.add_handler(auth_handler)
             opener.add_handler(http_handler)
             self._test_basic_auth(opener, auth_handler, "Authorization",
@@ -1544,7 +1579,7 @@ class HandlerTests(unittest.TestCase):
         password_manager = MockPasswordManager()
         auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager)
         realm = "ACME Networks"
-        http_handler = MockHTTPHandler(
+        http_handler = MockHTTPHandlerRedirect(
             407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
         opener.add_handler(auth_handler)
         opener.add_handler(http_handler)
@@ -1588,7 +1623,7 @@ class HandlerTests(unittest.TestCase):
         digest_handler = TestDigestAuthHandler(password_manager)
         basic_handler = TestBasicAuthHandler(password_manager)
         realm = "ACME Networks"
-        http_handler = MockHTTPHandler(
+        http_handler = MockHTTPHandlerRedirect(
             401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
         opener.add_handler(basic_handler)
         opener.add_handler(digest_handler)
@@ -1608,7 +1643,7 @@ class HandlerTests(unittest.TestCase):
         opener = OpenerDirector()
         # While using DigestAuthHandler
         digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None)
-        http_handler = MockHTTPHandler(
+        http_handler = MockHTTPHandlerRedirect(
             401, 'WWW-Authenticate: Kerberos\r\n\r\n')
         opener.add_handler(digest_auth_handler)
         opener.add_handler(http_handler)
@@ -1618,7 +1653,7 @@ class HandlerTests(unittest.TestCase):
         # While using BasicAuthHandler
         opener = OpenerDirector()
         basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None)
-        http_handler = MockHTTPHandler(
+        http_handler = MockHTTPHandlerRedirect(
             401, 'WWW-Authenticate: NTLM\r\n\r\n')
         opener.add_handler(basic_auth_handler)
         opener.add_handler(http_handler)
@@ -1705,7 +1740,7 @@ class HandlerTests(unittest.TestCase):
         opener = OpenerDirector()
         opener.add_handler(auth_prior_handler)
 
-        http_handler = MockHTTPHandler(
+        http_handler = MockHTTPHandlerRedirect(
             401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None)
         opener.add_handler(http_handler)
 

Lib/urllib/request.py

@@ -1251,8 +1251,8 @@ class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
 
 class AbstractHTTPHandler(BaseHandler):
 
-    def __init__(self, debuglevel=0):
+    def __init__(self, debuglevel=None):
-        self._debuglevel = debuglevel
+        self._debuglevel = debuglevel if debuglevel is not None else http.client.HTTPConnection.debuglevel
 
     def set_http_debuglevel(self, level):
         self._debuglevel = level
@@ -1378,7 +1378,8 @@ if hasattr(http.client, 'HTTPSConnection'):
 
     class HTTPSHandler(AbstractHTTPHandler):
 
-        def __init__(self, debuglevel=0, context=None, check_hostname=None):
+        def __init__(self, debuglevel=None, context=None, check_hostname=None):
+            debuglevel = debuglevel if debuglevel is not None else http.client.HTTPSConnection.debuglevel
             AbstractHTTPHandler.__init__(self, debuglevel)
             if context is None:
                 http_version = http.client.HTTPSConnection._http_vsn

Misc/NEWS.d/next/Library/2022-11-10-16-26-47.gh-issue-99353.DQFjnt.rst

@@ -0,0 +1,3 @@
+Respect the :class:`http.client.HTTPConnection` ``.debuglevel`` flag
+in :class:`urllib.request.AbstractHTTPHandler` when its constructor
+parameter ``debuglevel`` is not set. And do the same for ``*HTTPS*``.