Issue #16475: Support object instancing, recursion and interned strings

in marshal

Doc/library/marshal.rst

@@ -40,10 +40,11 @@ this module.  The following types are supported: booleans, integers, floating
 point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets,
 frozensets, dictionaries, and code objects, where it should be understood that
 tuples, lists, sets, frozensets and dictionaries are only supported as long as
-the values contained therein are themselves supported; and recursive lists, sets
+the values contained therein are themselves supported.
-and dictionaries should not be written (they will cause infinite loops).  The
 singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be
 marshalled and unmarshalled.
+For format *version* lower than 3, recursive lists, sets and dictionaries cannot
+be written (see below).
 
 There are functions that read/write files as well as functions operating on
 strings.
@@ -103,7 +104,9 @@ In addition, the following constants are defined:
 
    Indicates the format that the module uses. Version 0 is the historical
    format, version 1 shares interned strings and version 2 uses a binary format
-   for floating point numbers. The current version is 2.
+   for floating point numbers.
+   Version 3 adds support for object instancing and recursion.
+   The current version is 3.
 
 
 .. rubric:: Footnotes

Include/marshal.h

@@ -7,7 +7,7 @@
 extern "C" {
 #endif
 
-#define Py_MARSHAL_VERSION 2
+#define Py_MARSHAL_VERSION 3
 
 PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int);
 PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int);

Lib/test/test_marshal.py

@@ -200,10 +200,14 @@ class BugsTestCase(unittest.TestCase):
             except Exception:
                 pass
 
-    def test_loads_recursion(self):
+    def test_loads_2x_code(self):
         s = b'c' + (b'X' * 4*4) + b'{' * 2**20
         self.assertRaises(ValueError, marshal.loads, s)
 
+    def test_loads_recursion(self):
+        s = b'c' + (b'X' * 4*5) + b'{' * 2**20
+        self.assertRaises(ValueError, marshal.loads, s)
+
     def test_recursion_limit(self):
         # Create a deeply nested structure.
         head = last = []
@@ -323,6 +327,122 @@ class LargeValuesTestCase(unittest.TestCase):
     def test_bytearray(self, size):
         self.check_unmarshallable(bytearray(size))
 
+def CollectObjectIDs(ids, obj):
+    """Collect object ids seen in a structure"""
+    if id(obj) in ids:
+        return
+    ids.add(id(obj))
+    if isinstance(obj, (list, tuple, set, frozenset)):
+        for e in obj:
+            CollectObjectIDs(ids, e)
+    elif isinstance(obj, dict):
+        for k, v in obj.items():
+            CollectObjectIDs(ids, k)
+            CollectObjectIDs(ids, v)
+    return len(ids)
+
+class InstancingTestCase(unittest.TestCase, HelperMixin):
+    intobj = 123321
+    floatobj = 1.2345
+    strobj = "abcde"*3
+    dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"}
+
+    def helper3(self, rsample, recursive=False, simple=False):
+        #we have two instances
+        sample = (rsample, rsample)
+
+        n0 = CollectObjectIDs(set(), sample)
+
+        s3 = marshal.dumps(sample, 3)
+        n3 = CollectObjectIDs(set(), marshal.loads(s3))
+
+        #same number of instances generated
+        self.assertEqual(n3, n0)
+
+        if not recursive:
+            #can compare with version 2
+            s2 = marshal.dumps(sample, 2)
+            n2 = CollectObjectIDs(set(), marshal.loads(s2))
+            #old format generated more instances
+            self.assertGreater(n2, n0)
+
+            #if complex objects are in there, old format is larger
+            if not simple:
+                self.assertGreater(len(s2), len(s3))
+            else:
+                self.assertGreaterEqual(len(s2), len(s3))
+
+    def testInt(self):
+        self.helper(self.intobj)
+        self.helper3(self.intobj, simple=True)
+
+    def testFloat(self):
+        self.helper(self.floatobj)
+        self.helper3(self.floatobj)
+
+    def testStr(self):
+        self.helper(self.strobj)
+        self.helper3(self.strobj)
+
+    def testDict(self):
+        self.helper(self.dictobj)
+        self.helper3(self.dictobj)
+
+    def testModule(self):
+        with open(__file__, "rb") as f:
+            code = f.read()
+        if __file__.endswith(".py"):
+            code = compile(code, __file__, "exec")
+        self.helper(code)
+        self.helper3(code)
+
+    def testRecursion(self):
+        d = dict(self.dictobj)
+        d["self"] = d
+        self.helper3(d, recursive=True)
+        l = [self.dictobj]
+        l.append(l)
+        self.helper3(l, recursive=True)
+
+class CompatibilityTestCase(unittest.TestCase):
+    def _test(self, version):
+        with open(__file__, "rb") as f:
+            code = f.read()
+        if __file__.endswith(".py"):
+            code = compile(code, __file__, "exec")
+        data = marshal.dumps(code, version)
+        marshal.loads(data)
+
+    def test0To3(self):
+        self._test(0)
+
+    def test1To3(self):
+        self._test(1)
+
+    def test2To3(self):
+        self._test(2)
+
+    def test3To3(self):
+        self._test(3)
+
+class InterningTestCase(unittest.TestCase, HelperMixin):
+    strobj = "this is an interned string"
+    strobj = sys.intern(strobj)
+
+    def testIntern(self):
+        s = marshal.loads(marshal.dumps(self.strobj))
+        self.assertEqual(s, self.strobj)
+        self.assertEqual(id(s), id(self.strobj))
+        s2 = sys.intern(s)
+        self.assertEqual(id(s2), id(s))
+
+    def testNoIntern(self):
+        s = marshal.loads(marshal.dumps(self.strobj, 2))
+        self.assertEqual(s, self.strobj)
+        self.assertNotEqual(id(s), id(self.strobj))
+        s2 = sys.intern(s)
+        self.assertNotEqual(id(s2), id(s))
+
 
 def test_main():
     support.run_unittest(IntTestCase,

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.4.0 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #16475: Support object instancing, recursion and interned strings
+  in marshal
+
 - Use the HTTPS PyPI url for upload, overriding any plain HTTP URL in pypirc.
 
 - Issue #16795: On the ast.arguments object, unify vararg with varargannotation

Python/marshal.c

@@ -1,8 +1,10 @@
 
 /* Write Python objects to files and read them back.
-   This is intended for writing and reading compiled Python code only;
+   This is primarily intended for writing and reading compiled Python code,
-   a true persistent storage facility would be much harder, since
+   even though dicts, lists, sets and frozensets, not commonly seen in
-   it would have to take circular links and sharing into account. */
+   code objects, are supported.
+   Version 3 of this protocol properly supports circular links
+   and sharing. */
 
 #define PY_SSIZE_T_CLEAN
 
@@ -41,6 +43,8 @@
 #define TYPE_BINARY_COMPLEX     'y'
 #define TYPE_LONG               'l'
 #define TYPE_STRING             's'
+#define TYPE_INTERNED           't'
+#define TYPE_REF                'r'
 #define TYPE_TUPLE              '('
 #define TYPE_LIST               '['
 #define TYPE_DICT               '{'
@@ -49,6 +53,7 @@
 #define TYPE_UNKNOWN            '?'
 #define TYPE_SET                '<'
 #define TYPE_FROZENSET          '>'
+#define FLAG_REF                '\x80' /* with a type, add obj to index */
 
 #define WFERR_OK 0
 #define WFERR_UNMARSHALLABLE 1
@@ -65,6 +70,7 @@ typedef struct {
     PyObject *current_filename;
     char *ptr;
     char *end;
+    PyObject *refs; /* dict on marshal, list on unmarshal */
     int version;
 } WFILE;
 
@@ -151,13 +157,17 @@ w_long(long x, WFILE *p)
 #endif
 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
 
+#define W_TYPE(t, p) do { \
+    w_byte((t) | flag, (p)); \
+} while(0)
+
 static void
-w_PyLong(const PyLongObject *ob, WFILE *p)
+w_PyLong(const PyLongObject *ob, char flag, WFILE *p)
 {
     Py_ssize_t i, j, n, l;
     digit d;
 
-    w_byte(TYPE_LONG, p);
+    W_TYPE(TYPE_LONG, p);
     if (Py_SIZE(ob) == 0) {
         w_long((long)0, p);
         return;
@@ -194,10 +204,64 @@ w_PyLong(const PyLongObject *ob, WFILE *p)
     } while (d != 0);
 }
 
+static int
+w_ref(PyObject *v, char *flag, WFILE *p)
+{
+    PyObject *id;
+    PyObject *idx;
+
+    if (p->version < 3 || p->refs == NULL)
+        return 0; /* not writing object references */
+
+    /* if it has only one reference, it definitely isn't shared */
+    if (Py_REFCNT(v) == 1)
+        return 0;
+
+    id = PyLong_FromVoidPtr((void*)v);
+    if (id == NULL)
+        goto err;
+    idx = PyDict_GetItem(p->refs, id);
+    if (idx != NULL) {
+        /* write the reference index to the stream */
+        long w = PyLong_AsLong(idx);
+        Py_DECREF(id);
+        if (w == -1 && PyErr_Occurred()) {
+            goto err;
+        }
+        /* we don't store "long" indices in the dict */
+        assert(0 <= w && w <= 0x7fffffff);
+        w_byte(TYPE_REF, p);
+        w_long(w, p);
+        return 1;
+    } else {
+        int ok;
+        Py_ssize_t s = PyDict_Size(p->refs);
+        /* we don't support long indices */
+        if (s >= 0x7fffffff) {
+            PyErr_SetString(PyExc_ValueError, "too many objects");
+            goto err;
+        }
+        idx = PyLong_FromSsize_t(s);
+        ok = idx && PyDict_SetItem(p->refs, id, idx) == 0;
+        Py_DECREF(id);
+        Py_XDECREF(idx);
+        if (!ok)
+            goto err;
+        *flag |= FLAG_REF;
+        return 0;
+    }
+err:
+    p->error = WFERR_UNMARSHALLABLE;
+    return 1;
+}
+
+static void
+w_complex_object(PyObject *v, char flag, WFILE *p);
+
 static void
 w_object(PyObject *v, WFILE *p)
 {
-    Py_ssize_t i, n;
+    char flag = '\0';
 
     p->depth++;
 
@@ -222,24 +286,35 @@ w_object(PyObject *v, WFILE *p)
     else if (v == Py_True) {
         w_byte(TYPE_TRUE, p);
     }
-    else if (PyLong_CheckExact(v)) {
+    else if (!w_ref(v, &flag, p))
+        w_complex_object(v, flag, p);
+
+    p->depth--;
+}
+
+static void
+w_complex_object(PyObject *v, char flag, WFILE *p)
+{
+    Py_ssize_t i, n;
+
+    if (PyLong_CheckExact(v)) {
         long x = PyLong_AsLong(v);
         if ((x == -1)  && PyErr_Occurred()) {
             PyLongObject *ob = (PyLongObject *)v;
             PyErr_Clear();
-            w_PyLong(ob, p);
+            w_PyLong(ob, flag, p);
         }
         else {
 #if SIZEOF_LONG > 4
             long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
             if (y && y != -1) {
                 /* Too large for TYPE_INT */
-                w_PyLong((PyLongObject*)v, p);
+                w_PyLong((PyLongObject*)v, flag, p);
             }
             else
 #endif
             {
-                w_byte(TYPE_INT, p);
+                W_TYPE(TYPE_INT, p);
                 w_long(x, p);
             }
         }
@@ -252,7 +327,7 @@ w_object(PyObject *v, WFILE *p)
                 p->error = WFERR_UNMARSHALLABLE;
                 return;
             }
-            w_byte(TYPE_BINARY_FLOAT, p);
+            W_TYPE(TYPE_BINARY_FLOAT, p);
             w_string((char*)buf, 8, p);
         }
         else {
@@ -263,7 +338,7 @@ w_object(PyObject *v, WFILE *p)
                 return;
             }
             n = strlen(buf);
-            w_byte(TYPE_FLOAT, p);
+            W_TYPE(TYPE_FLOAT, p);
             w_byte((int)n, p);
             w_string(buf, n, p);
             PyMem_Free(buf);
@@ -277,7 +352,7 @@ w_object(PyObject *v, WFILE *p)
                 p->error = WFERR_UNMARSHALLABLE;
                 return;
             }
-            w_byte(TYPE_BINARY_COMPLEX, p);
+            W_TYPE(TYPE_BINARY_COMPLEX, p);
             w_string((char*)buf, 8, p);
             if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
                                buf, 1) < 0) {
@@ -288,7 +363,7 @@ w_object(PyObject *v, WFILE *p)
         }
         else {
             char *buf;
-            w_byte(TYPE_COMPLEX, p);
+            W_TYPE(TYPE_COMPLEX, p);
             buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
                                         'g', 17, 0, NULL);
             if (!buf) {
@@ -312,7 +387,7 @@ w_object(PyObject *v, WFILE *p)
         }
     }
     else if (PyBytes_CheckExact(v)) {
-        w_byte(TYPE_STRING, p);
+        W_TYPE(TYPE_STRING, p);
         n = PyBytes_GET_SIZE(v);
         W_SIZE(n, p);
         w_string(PyBytes_AS_STRING(v), n, p);
@@ -325,14 +400,17 @@ w_object(PyObject *v, WFILE *p)
             p->error = WFERR_UNMARSHALLABLE;
             return;
         }
-        w_byte(TYPE_UNICODE, p);
+        if (p->version >= 3 &&  PyUnicode_CHECK_INTERNED(v))
+            W_TYPE(TYPE_INTERNED, p);
+        else
+            W_TYPE(TYPE_UNICODE, p);
         n = PyBytes_GET_SIZE(utf8);
         W_SIZE(n, p);
         w_string(PyBytes_AS_STRING(utf8), n, p);
         Py_DECREF(utf8);
     }
     else if (PyTuple_CheckExact(v)) {
-        w_byte(TYPE_TUPLE, p);
+        W_TYPE(TYPE_TUPLE, p);
         n = PyTuple_Size(v);
         W_SIZE(n, p);
         for (i = 0; i < n; i++) {
@@ -340,7 +418,7 @@ w_object(PyObject *v, WFILE *p)
         }
     }
     else if (PyList_CheckExact(v)) {
-        w_byte(TYPE_LIST, p);
+        W_TYPE(TYPE_LIST, p);
         n = PyList_GET_SIZE(v);
         W_SIZE(n, p);
         for (i = 0; i < n; i++) {
@@ -350,7 +428,7 @@ w_object(PyObject *v, WFILE *p)
     else if (PyDict_CheckExact(v)) {
         Py_ssize_t pos;
         PyObject *key, *value;
-        w_byte(TYPE_DICT, p);
+        W_TYPE(TYPE_DICT, p);
         /* This one is NULL object terminated! */
         pos = 0;
         while (PyDict_Next(v, &pos, &key, &value)) {
@@ -363,9 +441,9 @@ w_object(PyObject *v, WFILE *p)
         PyObject *value, *it;
 
         if (PyObject_TypeCheck(v, &PySet_Type))
-            w_byte(TYPE_SET, p);
+            W_TYPE(TYPE_SET, p);
         else
-            w_byte(TYPE_FROZENSET, p);
+            W_TYPE(TYPE_FROZENSET, p);
         n = PyObject_Size(v);
         if (n == -1) {
             p->depth--;
@@ -392,7 +470,7 @@ w_object(PyObject *v, WFILE *p)
     }
     else if (PyCode_Check(v)) {
         PyCodeObject *co = (PyCodeObject *)v;
-        w_byte(TYPE_CODE, p);
+        W_TYPE(TYPE_CODE, p);
         w_long(co->co_argcount, p);
         w_long(co->co_kwonlyargcount, p);
         w_long(co->co_nlocals, p);
@@ -419,7 +497,7 @@ w_object(PyObject *v, WFILE *p)
             p->error = WFERR_UNMARSHALLABLE;
             return;
         }
-        w_byte(TYPE_STRING, p);
+        W_TYPE(TYPE_STRING, p);
         n = view.len;
         s = view.buf;
         W_SIZE(n, p);
@@ -427,10 +505,9 @@ w_object(PyObject *v, WFILE *p)
         PyBuffer_Release(&view);
     }
     else {
-        w_byte(TYPE_UNKNOWN, p);
+        W_TYPE(TYPE_UNKNOWN, p);
         p->error = WFERR_UNMARSHALLABLE;
     }
-    p->depth--;
 }
 
 /* version currently has no effect for writing longs. */
@@ -441,6 +518,7 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
     wf.fp = fp;
     wf.error = WFERR_OK;
     wf.depth = 0;
+    wf.refs = NULL;
     wf.version = version;
     w_long(x, &wf);
 }
@@ -452,8 +530,14 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
     wf.fp = fp;
     wf.error = WFERR_OK;
     wf.depth = 0;
+    if (version >= 3) {
+        if ((wf.refs = PyDict_New()) == NULL)
+            return; /* caller mush check PyErr_Occurred() */
+    } else
+        wf.refs = NULL;
     wf.version = version;
     w_object(x, &wf);
+    Py_XDECREF(wf.refs);
 }
 
 typedef WFILE RFILE; /* Same struct with different invariants */
@@ -489,7 +573,7 @@ r_string(char *s, Py_ssize_t n, RFILE *p)
                              data->ob_type->tp_name);
             }
             else {
-                read = PyBytes_GET_SIZE(data);
+                read = (int)PyBytes_GET_SIZE(data);
                 if (read > 0) {
                     ptr = PyBytes_AS_STRING(data);
                     memcpy(s, ptr, read);
@@ -659,6 +743,59 @@ r_PyLong(RFILE *p)
     return NULL;
 }
 
+/* allocate the reflist index */
+static PyObject *
+r_ref_reserve(PyObject *o, Py_ssize_t *idx, int flag, RFILE *p)
+{
+    if (flag) { /* currently only FLAG_REF is defined */
+        *idx = PyList_Size(p->refs);
+        if (*idx < 0)
+            goto err;
+        if (*idx >= 0x7ffffffe) {
+            PyErr_SetString(PyExc_ValueError, "bad marshal data (index list too large)");
+            goto err;
+        }
+        if (PyList_Append(p->refs, Py_None) < 0)
+            goto err;
+    } else
+        *idx = 0;
+    return o;
+err:
+    Py_XDECREF(o);  /* release the new object */
+    *idx = -1;
+    return NULL;
+}
+
+/* insert actual object to the reflist */
+static PyObject *
+r_ref_insert(PyObject *o, Py_ssize_t idx, int flag, RFILE *p)
+{
+    if (o && (flag & FLAG_REF)) {
+        if (PyList_SetItem(p->refs, idx, o) < 0) {
+            Py_DECREF(o); /* release the new object */
+            return NULL;
+        } else {
+            Py_INCREF(o); /* a reference for the list */
+        }
+    }
+    return o;
+}
+
+/* combination of both above, used when an object can be
+ * created whenever it is seen in the file, as opposed to
+ * after having loaded its sub-objects.
+ */
+static PyObject *
+r_ref(PyObject *o, int flag, RFILE *p)
+{
+    if (o && (flag & FLAG_REF)) {
+        if (PyList_Append(p->refs, o) < 0) {
+            Py_DECREF(o); /* release the new object */
+            return NULL;
+        }
+    }
+    return o;
+}
 
 static PyObject *
 r_object(RFILE *p)
@@ -666,8 +803,10 @@ r_object(RFILE *p)
     /* NULL is a valid return value, it does not necessarily means that
        an exception is set. */
     PyObject *v, *v2;
+    Py_ssize_t idx;
     long i, n;
     int type = r_byte(p);
+    int flag;
     PyObject *retval;
 
     p->depth++;
@@ -678,6 +817,13 @@ r_object(RFILE *p)
         return NULL;
     }
 
+    flag = type & FLAG_REF;
+    type = type & ~FLAG_REF;
+
+#define R_REF(O) do{\
+    O = r_ref(O, flag, p);\
+} while (0)
+
     switch (type) {
 
     case EOF:
@@ -718,14 +864,17 @@ r_object(RFILE *p)
     case TYPE_INT:
         n = r_long(p);
         retval = PyErr_Occurred() ? NULL : PyLong_FromLong(n);
+        R_REF(retval);
         break;
 
     case TYPE_INT64:
         retval = r_long64(p);
+        R_REF(retval);
         break;
 
     case TYPE_LONG:
         retval = r_PyLong(p);
+        R_REF(retval);
         break;
 
     case TYPE_FLOAT:
@@ -744,6 +893,7 @@ r_object(RFILE *p)
             if (dx == -1.0 && PyErr_Occurred())
                 break;
             retval = PyFloat_FromDouble(dx);
+            R_REF(retval);
             break;
         }
 
@@ -763,6 +913,7 @@ r_object(RFILE *p)
                 break;
             }
             retval = PyFloat_FromDouble(x);
+            R_REF(retval);
             break;
         }
 
@@ -792,6 +943,7 @@ r_object(RFILE *p)
             if (c.imag == -1.0 && PyErr_Occurred())
                 break;
             retval = PyComplex_FromCComplex(c);
+            R_REF(retval);
             break;
         }
 
@@ -822,6 +974,7 @@ r_object(RFILE *p)
                 break;
             }
             retval = PyComplex_FromCComplex(c);
+            R_REF(retval);
             break;
         }
 
@@ -849,9 +1002,11 @@ r_object(RFILE *p)
             break;
         }
         retval = v;
+        R_REF(retval);
         break;
 
     case TYPE_UNICODE:
+    case TYPE_INTERNED:
         {
         char *buffer;
 
@@ -879,7 +1034,10 @@ r_object(RFILE *p)
         }
         v = PyUnicode_DecodeUTF8(buffer, n, "surrogatepass");
         PyMem_DEL(buffer);
+        if (type == TYPE_INTERNED)
+            PyUnicode_InternInPlace(&v);
         retval = v;
+        R_REF(retval);
         break;
         }
 
@@ -895,6 +1053,7 @@ r_object(RFILE *p)
             break;
         }
         v = PyTuple_New(n);
+        R_REF(v);
         if (v == NULL) {
             retval = NULL;
             break;
@@ -926,6 +1085,7 @@ r_object(RFILE *p)
             break;
         }
         v = PyList_New(n);
+        R_REF(v);
         if (v == NULL) {
             retval = NULL;
             break;
@@ -947,6 +1107,7 @@ r_object(RFILE *p)
 
     case TYPE_DICT:
         v = PyDict_New();
+        R_REF(v);
         if (v == NULL) {
             retval = NULL;
             break;
@@ -982,6 +1143,13 @@ r_object(RFILE *p)
             break;
         }
         v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
+        /* must use delayed registration of frozensets because they must
+         * be init with a refcount of 1
+         */
+        if (type == TYPE_SET)
+            R_REF(v);
+        else
+            v = r_ref_reserve(v, &idx, flag, p);
         if (v == NULL) {
             retval = NULL;
             break;
@@ -1004,6 +1172,8 @@ r_object(RFILE *p)
             }
             Py_DECREF(v2);
         }
+        if (type != TYPE_SET)
+            v = r_ref_insert(v, idx, flag, p);
         retval = v;
         break;
 
@@ -1024,6 +1194,12 @@ r_object(RFILE *p)
             PyObject *name = NULL;
             int firstlineno;
             PyObject *lnotab = NULL;
+            
+            r_ref_reserve(NULL, &idx, flag, p);
+            if (idx < 0) {
+                retval = NULL;
+                break;
+            }
 
             v = NULL;
 
@@ -1090,6 +1266,7 @@ r_object(RFILE *p)
                             code, consts, names, varnames,
                             freevars, cellvars, filename, name,
                             firstlineno, lnotab);
+            v = r_ref_insert(v, idx, flag, p);
 
           code_error:
             Py_XDECREF(code);
@@ -1105,6 +1282,23 @@ r_object(RFILE *p)
         retval = v;
         break;
 
+    case TYPE_REF:
+        n = r_long(p);
+        if (n < 0 || n >= PyList_GET_SIZE(p->refs)) {
+            PyErr_SetString(PyExc_ValueError, "bad marshal data (invalid reference)");
+            retval = NULL;
+            break;
+        }
+        v = PyList_GET_ITEM(p->refs, n);
+        if (v == Py_None) {
+            PyErr_SetString(PyExc_ValueError, "bad marshal data (invalid reference)");
+            retval = NULL;
+            break;
+        }
+        Py_INCREF(v);
+        retval = v;
+        break;
+
     default:
         /* Bogus data got written, which isn't ideal.
            This will let you keep working and recover. */
@@ -1210,7 +1404,11 @@ PyMarshal_ReadObjectFromFile(FILE *fp)
     rf.current_filename = NULL;
     rf.depth = 0;
     rf.ptr = rf.end = NULL;
+    rf.refs = PyList_New(0);
+    if (rf.refs == NULL)
+        return NULL;
     result = r_object(&rf);
+    Py_DECREF(rf.refs);
     return result;
 }
 
@@ -1225,7 +1423,11 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
     rf.ptr = str;
     rf.end = str + len;
     rf.depth = 0;
+    rf.refs = PyList_New(0);
+    if (rf.refs == NULL)
+        return NULL;
     result = r_object(&rf);
+    Py_DECREF(rf.refs);
     return result;
 }
 
@@ -1244,7 +1446,13 @@ PyMarshal_WriteObjectToString(PyObject *x, int version)
     wf.error = WFERR_OK;
     wf.depth = 0;
     wf.version = version;
+    if (version >= 3) {
+        if ((wf.refs = PyDict_New()) == NULL)
+            return NULL;
+    } else
+        wf.refs = NULL;
     w_object(x, &wf);
+    Py_XDECREF(wf.refs);
     if (wf.str != NULL) {
         char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str);
         if (wf.ptr - base > PY_SSIZE_T_MAX) {
@@ -1316,6 +1524,8 @@ marshal_load(PyObject *self, PyObject *f)
      * Make a call to the read method, but read zero bytes.
      * This is to ensure that the object passed in at least
      * has a read method which returns bytes.
+     * This can be removed if we guarantee good error handling
+     * for r_string()
      */
     data = _PyObject_CallMethodId(f, &PyId_read, "i", 0);
     if (data == NULL)
@@ -1331,7 +1541,11 @@ marshal_load(PyObject *self, PyObject *f)
         rf.fp = NULL;
         rf.readable = f;
         rf.current_filename = NULL;
-        result = read_object(&rf);
+        if ((rf.refs = PyList_New(0)) != NULL) {
+            result = read_object(&rf);
+            Py_DECREF(rf.refs);
+        } else
+            result = NULL;
     }
     Py_DECREF(data);
     return result;
@@ -1388,8 +1602,11 @@ marshal_loads(PyObject *self, PyObject *args)
     rf.ptr = s;
     rf.end = s + n;
     rf.depth = 0;
+    if ((rf.refs = PyList_New(0)) == NULL)
+        return NULL;
     result = read_object(&rf);
     PyBuffer_Release(&p);
+    Py_DECREF(rf.refs);
     return result;
 }
 
@@ -1429,6 +1646,7 @@ Variables:\n\
 version -- indicates the format that the module uses. Version 0 is the\n\
     historical format, version 1 shares interned strings and version 2\n\
     uses a binary format for floating point numbers.\n\
+    Version 3 shares common object references (New in version 3.4).\n\
 \n\
 Functions:\n\
 \n\