bpo-40014: Fix os.getgrouplist() (GH-19126)

Fix os.getgrouplist(): if getgrouplist() function fails because the
group list is too small, retry with a larger group list.

On failure, the glibc implementation of getgrouplist() sets ngroups
to the total number of groups. For other implementations, double the
group list size.

Misc/NEWS.d/next/Library/2020-03-23-17-52-00.bpo-40014.Ya70VG.rst

@@ -1,3 +1,4 @@
-Fix ``os.getgrouplist()``: on macOS, the ``getgrouplist()`` function returns a
+Fix ``os.getgrouplist()``: if ``getgrouplist()`` function fails because the
-non-zero value without setting ``errno`` if the group list is too small. Double
+group list is too small, retry with a larger group list. On failure, the glibc
-the list size and call it again in this case.
+implementation of ``getgrouplist()`` sets ``ngroups`` to the total number of
+groups. For other implementations, double the group list size.

Modules/posixmodule.c

@@ -6991,37 +6991,40 @@ posix_getgrouplist(PyObject *self, PyObject *args)
         return NULL;
 #endif
 
+    while (1) {
 #ifdef __APPLE__
-    groups = PyMem_New(int, ngroups);
-#else
-    groups = PyMem_New(gid_t, ngroups);
-#endif
-    if (groups == NULL)
-        return PyErr_NoMemory();
-
-#ifdef __APPLE__
-    while (getgrouplist(user, basegid, groups, &ngroups)) {
-        /* On macOS, getgrouplist() returns a non-zero value without setting
-           errno if the group list is too small. Double the list size and call
-           it again in this case. */
-        PyMem_Free(groups);
-
-        if (ngroups > INT_MAX / 2) {
-            return PyErr_NoMemory();
-        }
-        ngroups *= 2;
-
         groups = PyMem_New(int, ngroups);
+#else
+        groups = PyMem_New(gid_t, ngroups);
+#endif
         if (groups == NULL) {
             return PyErr_NoMemory();
         }
+
+        int old_ngroups = ngroups;
+        if (getgrouplist(user, basegid, groups, &ngroups) != -1) {
+            /* Success */
+            break;
+        }
+
+        /* getgrouplist() fails if the group list is too small */
+        PyMem_Free(groups);
+
+        if (ngroups > old_ngroups) {
+            /* If the group list is too small, the glibc implementation of
+               getgrouplist() sets ngroups to the total number of groups and
+               returns -1. */
+        }
+        else {
+            /* Double the group list size */
+            if (ngroups > INT_MAX / 2) {
+                return PyErr_NoMemory();
+            }
+            ngroups *= 2;
+        }
+
+        /* Retry getgrouplist() with a larger group list */
     }
-#else
-    if (getgrouplist(user, basegid, groups, &ngroups) == -1) {
-        PyMem_Del(groups);
-        return posix_error();
-    }
-#endif
 
 #ifdef _Py_MEMORY_SANITIZER
     /* Clang memory sanitizer libc intercepts don't know getgrouplist. */