[ruby/openssl] Add X509::Certificate#tbs_bytes

Ref https://github.com/ruby/openssl/issues/519

This makes verifying embedded certificate transparency signatures
significantly easier, as otherwise the alternative was manipulating the
ASN1 sequence, as in
656d992fa8

https://github.com/ruby/openssl/commit/99128bea5d

ext/openssl/extconf.rb

@@ -149,6 +149,9 @@ engines.each { |name|
   have_func("ENGINE_load_#{name}()", "openssl/engine.h")
 }
 
+# missing in libressl < 3.5
+have_func("i2d_re_X509_tbs(NULL, NULL)", x509_h)
+
 # added in 1.1.0
 if !have_struct_member("SSL", "ctx", "openssl/ssl.h") || is_libressl
   $defs.push("-DHAVE_OPAQUE_OPENSSL")

ext/openssl/ossl_x509cert.c

@@ -707,6 +707,38 @@ ossl_x509_eq(VALUE self, VALUE other)
     return !X509_cmp(a, b) ? Qtrue : Qfalse;
 }
 
+#ifdef HAVE_I2D_RE_X509_TBS
+/*
+ * call-seq:
+ *    cert.tbs_bytes => string
+ *
+ * Returns the DER-encoded bytes of the certificate's to be signed certificate.
+ * This is mainly useful for validating embedded certificate transparency signatures.
+ */
+static VALUE
+ossl_x509_tbs_bytes(VALUE self)
+{
+    X509 *x509;
+    int len;
+    unsigned char *p0;
+    VALUE str;
+
+    GetX509(self, x509);
+    len = i2d_re_X509_tbs(x509, NULL);
+    if (len <= 0) {
+        ossl_raise(eX509CertError, "i2d_re_X509_tbs");
+    }
+    str = rb_str_new(NULL, len);
+    p0 = (unsigned char *)RSTRING_PTR(str);
+    if (i2d_re_X509_tbs(x509, &p0) <= 0) {
+        ossl_raise(eX509CertError, "i2d_re_X509_tbs");
+    }
+    ossl_str_adjust(str, p0);
+
+    return str;
+}
+#endif
+
 struct load_chained_certificates_arguments {
     VALUE certificates;
     X509 *certificate;
@@ -999,4 +1031,7 @@ Init_ossl_x509cert(void)
     rb_define_method(cX509Cert, "add_extension", ossl_x509_add_extension, 1);
     rb_define_method(cX509Cert, "inspect", ossl_x509_inspect, 0);
     rb_define_method(cX509Cert, "==", ossl_x509_eq, 1);
+#ifdef HAVE_I2D_RE_X509_TBS
+    rb_define_method(cX509Cert, "tbs_bytes", ossl_x509_tbs_bytes, 0);
+#endif
 }

test/openssl/test_x509cert.rb

@@ -322,6 +322,15 @@ class OpenSSL::TestX509Certificate < OpenSSL::TestCase
     end
   end
 
+  def test_tbs_precert_bytes
+    pend "LibreSSL < 3.5 does not have i2d_re_X509_tbs" if libressl? && !libressl?(3, 5, 0)
+
+    cert = issue_cert(@ca, @rsa2048, 1, [], nil, nil)
+    seq = OpenSSL::ASN1.decode(cert.tbs_bytes)
+
+    assert_equal 7, seq.value.size
+  end
+
   private
 
   def certificate_error_returns_false