From 34b407a4a89e69dd04f692e2b29efa2816d4664a Mon Sep 17 00:00:00 2001
From: Peter Zhu <peter@peterzhu.ca>
Date: Mon, 2 Jun 2025 15:02:59 -0400
Subject: [PATCH] Fix memory leak in Prism's RubyVM::InstructionSequence.new

[Bug #21394]

There are two ways to make RubyVM::InstructionSequence.new raise which
would cause the options->scopes to leak memory:

1. Passing in any (non T_FILE) object where the to_str raises.
2. Passing in a T_FILE object where String#initialize_dup raises. This is
   because rb_io_path dups the string.

Example 1:

    10.times do
      100_000.times do
        RubyVM::InstructionSequence.new(nil)
      rescue TypeError
      end

      puts `ps -o rss= -p #{$$}`
    end

Before:

    13392
    17104
    20256
    23920
    27264
    30432
    33584
    36752
    40032
    43232

After:

    9392
    11072
    11648
    11648
    11648
    11712
    11712
    11712
    11744
    11744

Example 2:

    require "tempfile"

    MyError = Class.new(StandardError)
    String.prepend(Module.new do
      def initialize_dup(_)
        if $raise_on_dup
          raise MyError
        else
          super
        end
      end
    end)

    Tempfile.create do |f|
      10.times do
        100_000.times do
          $raise_on_dup = true
          RubyVM::InstructionSequence.new(f)
        rescue MyError
        else
          raise "MyError was not raised during RubyVM::InstructionSequence.new"
        end

        puts `ps -o rss= -p #{$$}`
      ensure
        $raise_on_dup = false
      end
    end

Before:

    14080
    18512
    22000
    25184
    28320
    31600
    34736
    37904
    41088
    44256

After:

    12016
    12464
    12880
    12880
    12880
    12912
    12912
    12912
    12912
    12912
---
 iseq.c                 | 18 ++++++++++-----
 test/ruby/test_iseq.rb | 50 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/iseq.c b/iseq.c
index c8c2c6846f..dcde27ba1b 100644
--- a/iseq.c
+++ b/iseq.c
@@ -1327,6 +1327,15 @@ pm_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V
     ln = NUM2INT(line);
     StringValueCStr(file);
 
+    bool parse_file = false;
+    if (RB_TYPE_P(src, T_FILE)) {
+        parse_file = true;
+        src = rb_io_path(src);
+    }
+    else {
+        src = StringValue(src);
+    }
+
     pm_parse_result_t result = { 0 };
     pm_options_line_set(&result.options, NUM2INT(line));
     pm_options_scopes_init(&result.options, 1);
@@ -1349,16 +1358,15 @@ pm_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V
     VALUE script_lines;
     VALUE error;
 
-    if (RB_TYPE_P(src, T_FILE)) {
-        VALUE filepath = rb_io_path(src);
-        error = pm_load_parse_file(&result, filepath, ruby_vm_keep_script_lines ? &script_lines : NULL);
-        RB_GC_GUARD(filepath);
+    if (parse_file) {
+        error = pm_load_parse_file(&result, src, ruby_vm_keep_script_lines ? &script_lines : NULL);
     }
     else {
-        src = StringValue(src);
         error = pm_parse_string(&result, src, file, ruby_vm_keep_script_lines ? &script_lines : NULL);
     }
 
+    RB_GC_GUARD(src);
+
     if (error == Qnil) {
         int error_state;
         iseq = pm_iseq_new_with_opt(&result.node, name, file, realpath, ln, NULL, 0, ISEQ_TYPE_TOP, &option, &error_state);
diff --git a/test/ruby/test_iseq.rb b/test/ruby/test_iseq.rb
index 29c8b1bf2d..45223c89da 100644
--- a/test/ruby/test_iseq.rb
+++ b/test/ruby/test_iseq.rb
@@ -297,6 +297,56 @@ class TestISeq < Test::Unit::TestCase
     assert_raise(TypeError, bug11159) {compile(1)}
   end
 
+  def test_invalid_source_no_memory_leak
+    # [Bug #21394]
+    assert_no_memory_leak(["-rtempfile"], "#{<<-"begin;"}", "#{<<-'end;'}", rss: true)
+      code = proc do |t|
+        RubyVM::InstructionSequence.new(nil)
+      rescue TypeError
+      else
+        raise "TypeError was not raised during RubyVM::InstructionSequence.new"
+      end
+
+      10.times(&code)
+    begin;
+      1_000_000.times(&code)
+    end;
+
+    # [Bug #21394]
+    # RubyVM::InstructionSequence.new calls rb_io_path, which dups the string
+    # and can leak memory if the dup raises
+    assert_no_memory_leak(["-rtempfile"], "#{<<-"begin;"}", "#{<<-'end;'}", rss: true)
+      MyError = Class.new(StandardError)
+      String.prepend(Module.new do
+        def initialize_dup(_)
+          if $raise_on_dup
+            raise MyError
+          else
+            super
+          end
+        end
+      end)
+
+      code = proc do |t|
+        Tempfile.create do |f|
+          $raise_on_dup = true
+          t.times do
+            RubyVM::InstructionSequence.new(f)
+          rescue MyError
+          else
+            raise "MyError was not raised during RubyVM::InstructionSequence.new"
+          end
+        ensure
+          $raise_on_dup = false
+        end
+      end
+
+      code.call(100)
+    begin;
+      code.call(1_000_000)
+    end;
+  end
+
   def test_frozen_string_literal_compile_option
     $f = 'f'
     line = __LINE__ + 2