ruby/lib/ruby_vm/rjit/insn_compiler.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

5248 lines
175 KiB
Ruby
Raw Normal View History

2023-03-06 23:15:30 -08:00
module RubyVM::RJIT
2022-12-26 14:09:45 -08:00
class InsnCompiler
# struct rb_calling_info. Storing flags instead of ci.
CallingInfo = Struct.new(:argc, :flags, :kwarg, :send_shift, :block_handler) do
def kw_splat = flags & C::VM_CALL_KW_SPLAT != 0
end
2022-12-30 22:16:07 -08:00
# @param ocb [CodeBlock]
2023-03-06 23:15:30 -08:00
# @param exit_compiler [RubyVM::RJIT::ExitCompiler]
2023-02-03 22:42:13 -08:00
def initialize(cb, ocb, exit_compiler)
2022-12-30 22:16:07 -08:00
@ocb = ocb
2023-01-02 14:11:06 -08:00
@exit_compiler = exit_compiler
2023-02-10 11:43:53 -08:00
2023-03-02 22:23:47 -08:00
@cfunc_codegen_table = {}
register_cfunc_codegen_funcs
2022-12-30 21:27:12 -08:00
end
2022-12-28 13:50:24 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
# @param insn `RubyVM::RJIT::Instruction`
def compile(jit, ctx, asm, insn)
2023-03-06 23:17:25 -08:00
asm.incr_counter(:rjit_insns_count)
asm.comment("Insn: #{insn.name}")
2023-03-18 21:49:42 -07:00
# 83/102
case insn.name
2023-02-07 23:40:05 -08:00
when :nop then nop(jit, ctx, asm)
2023-02-14 22:24:32 -08:00
when :getlocal then getlocal(jit, ctx, asm)
when :setlocal then setlocal(jit, ctx, asm)
2023-03-03 23:00:30 -08:00
when :getblockparam then getblockparam(jit, ctx, asm)
# setblockparam
2023-03-03 21:29:20 -08:00
when :getblockparamproxy then getblockparamproxy(jit, ctx, asm)
2023-03-18 00:00:18 -07:00
when :getspecial then getspecial(jit, ctx, asm)
# setspecial
2023-02-07 14:42:58 -08:00
when :getinstancevariable then getinstancevariable(jit, ctx, asm)
2023-02-08 17:27:21 -08:00
when :setinstancevariable then setinstancevariable(jit, ctx, asm)
2023-03-03 21:41:35 -08:00
when :getclassvariable then getclassvariable(jit, ctx, asm)
2023-03-18 21:49:42 -07:00
when :setclassvariable then setclassvariable(jit, ctx, asm)
when :opt_getconstant_path then opt_getconstant_path(jit, ctx, asm)
2023-03-03 22:49:31 -08:00
when :getconstant then getconstant(jit, ctx, asm)
# setconstant
2023-03-18 21:20:37 -07:00
when :getglobal then getglobal(jit, ctx, asm)
# setglobal
when :putnil then putnil(jit, ctx, asm)
2023-01-07 13:39:33 -08:00
when :putself then putself(jit, ctx, asm)
when :putobject then putobject(jit, ctx, asm)
2023-03-17 23:34:36 -07:00
when :putspecialobject then putspecialobject(jit, ctx, asm)
2023-02-21 00:19:37 -08:00
when :putstring then putstring(jit, ctx, asm)
2023-03-03 22:21:56 -08:00
when :concatstrings then concatstrings(jit, ctx, asm)
2023-03-03 22:17:34 -08:00
when :anytostring then anytostring(jit, ctx, asm)
2023-03-18 21:39:24 -07:00
when :toregexp then toregexp(jit, ctx, asm)
2023-03-18 21:42:10 -07:00
when :intern then intern(jit, ctx, asm)
2023-02-21 00:02:56 -08:00
when :newarray then newarray(jit, ctx, asm)
# newarraykwsplat
2023-02-15 21:26:04 -08:00
when :duparray then duparray(jit, ctx, asm)
# duphash
2023-02-15 21:26:04 -08:00
when :expandarray then expandarray(jit, ctx, asm)
2023-03-03 22:54:24 -08:00
when :concatarray then concatarray(jit, ctx, asm)
2023-02-21 00:16:05 -08:00
when :splatarray then splatarray(jit, ctx, asm)
2023-02-21 00:32:37 -08:00
when :newhash then newhash(jit, ctx, asm)
2023-03-18 21:24:28 -07:00
when :newrange then newrange(jit, ctx, asm)
2023-02-08 17:33:27 -08:00
when :pop then pop(jit, ctx, asm)
2023-02-08 17:17:36 -08:00
when :dup then dup(jit, ctx, asm)
2023-02-14 22:52:12 -08:00
when :dupn then dupn(jit, ctx, asm)
2023-02-21 00:22:50 -08:00
when :swap then swap(jit, ctx, asm)
# opt_reverse
2023-02-21 00:11:56 -08:00
when :topn then topn(jit, ctx, asm)
2023-02-14 22:52:12 -08:00
when :setn then setn(jit, ctx, asm)
2023-02-20 23:57:35 -08:00
when :adjuststack then adjuststack(jit, ctx, asm)
2023-02-28 22:45:59 -08:00
when :defined then defined(jit, ctx, asm)
when :definedivar then definedivar(jit, ctx, asm)
# checkmatch
2023-03-18 21:14:35 -07:00
when :checkkeyword then checkkeyword(jit, ctx, asm)
# checktype
# defineclass
# definemethod
# definesmethod
2023-02-28 22:50:04 -08:00
when :send then send(jit, ctx, asm)
2023-01-07 21:24:30 -08:00
when :opt_send_without_block then opt_send_without_block(jit, ctx, asm)
2023-03-03 22:03:23 -08:00
when :objtostring then objtostring(jit, ctx, asm)
2023-03-03 22:45:25 -08:00
when :opt_str_freeze then opt_str_freeze(jit, ctx, asm)
when :opt_nil_p then opt_nil_p(jit, ctx, asm)
# opt_str_uminus
# opt_newarray_max
2023-03-01 23:38:12 -08:00
when :opt_newarray_min then opt_newarray_min(jit, ctx, asm)
2023-02-24 14:48:02 -08:00
when :invokesuper then invokesuper(jit, ctx, asm)
2023-03-12 22:27:43 -07:00
when :invokeblock then invokeblock(jit, ctx, asm)
when :leave then leave(jit, ctx, asm)
2023-03-17 23:27:16 -07:00
when :throw then throw(jit, ctx, asm)
2023-02-08 17:42:16 -08:00
when :jump then jump(jit, ctx, asm)
2023-02-13 07:30:25 -08:00
when :branchif then branchif(jit, ctx, asm)
2023-01-07 13:21:14 -08:00
when :branchunless then branchunless(jit, ctx, asm)
2023-03-03 22:42:03 -08:00
when :branchnil then branchnil(jit, ctx, asm)
# once
2023-03-03 21:10:17 -08:00
when :opt_case_dispatch then opt_case_dispatch(jit, ctx, asm)
2023-02-06 15:44:34 -08:00
when :opt_plus then opt_plus(jit, ctx, asm)
2023-01-07 14:06:38 -08:00
when :opt_minus then opt_minus(jit, ctx, asm)
when :opt_mult then opt_mult(jit, ctx, asm)
when :opt_div then opt_div(jit, ctx, asm)
2023-02-13 22:36:02 -08:00
when :opt_mod then opt_mod(jit, ctx, asm)
2023-02-13 23:57:40 -08:00
when :opt_eq then opt_eq(jit, ctx, asm)
when :opt_neq then opt_neq(jit, ctx, asm)
when :opt_lt then opt_lt(jit, ctx, asm)
2023-02-13 21:48:24 -08:00
when :opt_le then opt_le(jit, ctx, asm)
when :opt_gt then opt_gt(jit, ctx, asm)
when :opt_ge then opt_ge(jit, ctx, asm)
when :opt_ltlt then opt_ltlt(jit, ctx, asm)
2023-02-13 23:05:56 -08:00
when :opt_and then opt_and(jit, ctx, asm)
when :opt_or then opt_or(jit, ctx, asm)
2023-02-08 11:10:04 -08:00
when :opt_aref then opt_aref(jit, ctx, asm)
2023-02-15 23:13:14 -08:00
when :opt_aset then opt_aset(jit, ctx, asm)
# opt_aset_with
# opt_aref_with
when :opt_length then opt_length(jit, ctx, asm)
when :opt_size then opt_size(jit, ctx, asm)
when :opt_empty_p then opt_empty_p(jit, ctx, asm)
when :opt_succ then opt_succ(jit, ctx, asm)
when :opt_not then opt_not(jit, ctx, asm)
when :opt_regexpmatch2 then opt_regexpmatch2(jit, ctx, asm)
# invokebuiltin
2023-02-16 00:11:38 -08:00
when :opt_invokebuiltin_delegate then opt_invokebuiltin_delegate(jit, ctx, asm)
when :opt_invokebuiltin_delegate_leave then opt_invokebuiltin_delegate_leave(jit, ctx, asm)
when :getlocal_WC_0 then getlocal_WC_0(jit, ctx, asm)
2023-02-08 01:17:39 -08:00
when :getlocal_WC_1 then getlocal_WC_1(jit, ctx, asm)
2023-02-13 22:50:52 -08:00
when :setlocal_WC_0 then setlocal_WC_0(jit, ctx, asm)
2023-02-14 22:24:32 -08:00
when :setlocal_WC_1 then setlocal_WC_1(jit, ctx, asm)
2023-01-07 13:54:26 -08:00
when :putobject_INT2FIX_0_ then putobject_INT2FIX_0_(jit, ctx, asm)
when :putobject_INT2FIX_1_ then putobject_INT2FIX_1_(jit, ctx, asm)
else CantCompile
end
end
private
#
# Insns
#
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-07 23:40:05 -08:00
def nop(jit, ctx, asm)
# Do nothing
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-14 22:24:32 -08:00
def getlocal(jit, ctx, asm)
idx = jit.operand(0)
level = jit.operand(1)
jit_getlocal_generic(jit, ctx, asm, idx:, level:)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-14 22:24:32 -08:00
def setlocal(jit, ctx, asm)
idx = jit.operand(0)
level = jit.operand(1)
jit_setlocal_generic(jit, ctx, asm, idx:, level:)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:00:30 -08:00
def getblockparam(jit, ctx, asm)
# EP level
level = jit.operand(1)
# Save the PC and SP because we might allocate
jit_prepare_routine_call(jit, ctx, asm)
# A mirror of the interpreter code. Checking for the case
# where it's pushing rb_block_param_proxy.
side_exit = side_exit(jit, ctx)
# Load environment pointer EP from CFP
ep_reg = :rax
jit_get_ep(asm, level, reg: ep_reg)
# Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero
# FIXME: This is testing bits in the same place that the WB check is testing.
# We should combine these at some point
asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)
2023-03-03 23:00:30 -08:00
# If the frame flag has been modified, then the actual proc value is
# already in the EP and we should just use the value.
frame_flag_modified = asm.new_label('frame_flag_modified')
asm.jnz(frame_flag_modified)
# This instruction writes the block handler to the EP. If we need to
# fire a write barrier for the write, then exit (we'll let the
# interpreter handle it so it can fire the write barrier).
# flags & VM_ENV_FLAG_WB_REQUIRED
asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_ENV_FLAG_WB_REQUIRED)
2023-03-03 23:00:30 -08:00
# if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0
asm.jnz(side_exit)
# Convert the block handler in to a proc
# call rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)
asm.mov(C_ARGS[0], EC)
# The block handler for the current frame
# note, VM_ASSERT(VM_ENV_LOCAL_P(ep))
asm.mov(C_ARGS[1], [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL])
2023-03-03 23:00:30 -08:00
asm.call(C.rb_vm_bh_to_procval)
# Load environment pointer EP from CFP (again)
ep_reg = :rcx
jit_get_ep(asm, level, reg: ep_reg)
# Write the value at the environment pointer
idx = jit.operand(0)
offs = -(C.VALUE.size * idx)
asm.mov([ep_reg, offs], C_RET);
# Set the frame modified flag
asm.mov(:rax, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS]) # flag_check
asm.or(:rax, C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) # modified_flag
asm.mov([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], :rax)
2023-03-03 23:00:30 -08:00
asm.write_label(frame_flag_modified)
# Push the proc on the stack
stack_ret = ctx.stack_push
ep_reg = :rax
jit_get_ep(asm, level, reg: ep_reg)
asm.mov(:rax, [ep_reg, offs])
asm.mov(stack_ret, :rax)
KeepCompiling
end
2022-12-28 13:50:24 -08:00
# setblockparam
2023-03-03 21:29:20 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 21:29:20 -08:00
def getblockparamproxy(jit, ctx, asm)
# To get block_handler
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
starting_context = ctx.dup # make a copy for use with jit_chain_guard
# A mirror of the interpreter code. Checking for the case
# where it's pushing rb_block_param_proxy.
side_exit = side_exit(jit, ctx)
# EP level
level = jit.operand(1)
# Peek at the block handler so we can check whether it's nil
comptime_handler = jit.peek_at_block_handler(level)
# When a block handler is present, it should always be a GC-guarded
# pointer (VM_BH_ISEQ_BLOCK_P)
if comptime_handler != 0 && comptime_handler & 0x3 != 0x1
asm.incr_counter(:getblockpp_not_gc_guarded)
return CantCompile
end
# Load environment pointer EP from CFP
ep_reg = :rax
jit_get_ep(asm, level, reg: ep_reg)
# Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero
asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)
2023-03-03 21:29:20 -08:00
asm.jnz(counted_exit(side_exit, :getblockpp_block_param_modified))
# Load the block handler for the current frame
# note, VM_ASSERT(VM_ENV_LOCAL_P(ep))
block_handler = :rax
asm.mov(block_handler, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL])
2023-03-03 21:29:20 -08:00
# Specialize compilation for the case where no block handler is present
if comptime_handler == 0
# Bail if there is a block handler
asm.cmp(block_handler, 0)
jit_chain_guard(:jnz, jit, starting_context, asm, counted_exit(side_exit, :getblockpp_block_handler_none))
putobject(jit, ctx, asm, val: Qnil)
else
# Block handler is a tagged pointer. Look at the tag. 0x03 is from VM_BH_ISEQ_BLOCK_P().
asm.and(block_handler, 0x3)
# Bail unless VM_BH_ISEQ_BLOCK_P(bh). This also checks for null.
asm.cmp(block_handler, 0x1)
jit_chain_guard(:jnz, jit, starting_context, asm, counted_exit(side_exit, :getblockpp_not_iseq_block))
# Push rb_block_param_proxy. It's a root, so no need to use jit_mov_gc_ptr.
top = ctx.stack_push
asm.mov(:rax, C.rb_block_param_proxy)
asm.mov(top, :rax)
end
jump_to_next_insn(jit, ctx, asm)
EndBlock
end
2023-03-18 00:00:18 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def getspecial(jit, ctx, asm)
# This takes two arguments, key and type
# key is only used when type == 0
# A non-zero type determines which type of backref to fetch
#rb_num_t key = jit.jit_get_arg(0);
rtype = jit.operand(1)
if rtype == 0
# not yet implemented
return CantCompile;
elsif rtype & 0x01 != 0
# Fetch a "special" backref based on a char encoded by shifting by 1
# Can raise if matchdata uninitialized
jit_prepare_routine_call(jit, ctx, asm)
# call rb_backref_get()
asm.comment('rb_backref_get')
asm.call(C.rb_backref_get)
asm.mov(C_ARGS[0], C_RET) # backref
case [rtype >> 1].pack('c')
in ?&
asm.comment("rb_reg_last_match")
asm.call(C.rb_reg_last_match)
in ?`
asm.comment("rb_reg_match_pre")
asm.call(C.rb_reg_match_pre)
in ?'
asm.comment("rb_reg_match_post")
asm.call(C.rb_reg_match_post)
in ?+
asm.comment("rb_reg_match_last")
asm.call(C.rb_reg_match_last)
end
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
else
# Fetch the N-th match from the last backref based on type shifted by 1
# Can raise if matchdata uninitialized
jit_prepare_routine_call(jit, ctx, asm)
# call rb_backref_get()
asm.comment('rb_backref_get')
asm.call(C.rb_backref_get)
# rb_reg_nth_match((int)(type >> 1), backref);
asm.comment('rb_reg_nth_match')
asm.mov(C_ARGS[0], rtype >> 1)
asm.mov(C_ARGS[1], C_RET) # backref
asm.call(C.rb_reg_nth_match)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
end
2022-12-28 13:50:24 -08:00
# setspecial
2023-02-07 14:42:58 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-07 14:42:58 -08:00
def getinstancevariable(jit, ctx, asm)
2023-02-08 01:48:32 -08:00
# Specialize on a compile-time receiver, and split a block for chain guards
2023-02-07 14:42:58 -08:00
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
id = jit.operand(0)
comptime_obj = jit.peek_at_self
jit_getivar(jit, ctx, asm, comptime_obj, id)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 17:27:21 -08:00
def setinstancevariable(jit, ctx, asm)
2023-03-02 21:14:26 -08:00
starting_context = ctx.dup # make a copy for use with jit_chain_guard
2023-02-08 17:27:21 -08:00
2023-03-02 21:14:26 -08:00
# Defer compilation so we can specialize on a runtime `self`
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
2023-02-08 17:27:21 -08:00
2023-03-02 21:14:26 -08:00
ivar_name = jit.operand(0)
comptime_receiver = jit.peek_at_self
2023-02-08 17:27:21 -08:00
2023-03-02 21:14:26 -08:00
# If the comptime receiver is frozen, writing an IV will raise an exception
# and we don't want to JIT code to deal with that situation.
if C.rb_obj_frozen_p(comptime_receiver)
asm.incr_counter(:setivar_frozen)
return CantCompile
end
# Check if the comptime receiver is a T_OBJECT
receiver_t_object = C::BUILTIN_TYPE(comptime_receiver) == C::T_OBJECT
2023-03-02 21:14:26 -08:00
# If the receiver isn't a T_OBJECT, or uses a custom allocator,
# then just write out the IV write as a function call.
# too-complex shapes can't use index access, so we use rb_ivar_get for them too.
if !receiver_t_object || shape_too_complex?(comptime_receiver) || ctx.chain_depth >= 10
asm.comment('call rb_vm_setinstancevariable')
ic = jit.operand(1)
# The function could raise exceptions.
# Note that this modifies REG_SP, which is why we do it first
jit_prepare_routine_call(jit, ctx, asm)
# Get the operands from the stack
val_opnd = ctx.stack_pop(1)
# Call rb_vm_setinstancevariable(iseq, obj, id, val, ic);
asm.mov(:rdi, jit.iseq.to_i)
asm.mov(:rsi, [CFP, C.rb_control_frame_t.offsetof(:self)])
asm.mov(:rdx, ivar_name)
asm.mov(:rcx, val_opnd)
asm.mov(:r8, ic)
asm.call(C.rb_vm_setinstancevariable)
else
# Get the iv index
shape_id = C.rb_shape_get_shape_id(comptime_receiver)
ivar_index = C.rb_shape_get_iv_index(shape_id, ivar_name)
# Get the receiver
asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)])
# Generate a side exit
side_exit = side_exit(jit, ctx)
# Upgrade type
guard_object_is_heap(asm, :rax, counted_exit(side_exit, :setivar_not_heap))
asm.comment('guard shape')
asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id)
megamorphic_side_exit = counted_exit(side_exit, :setivar_megamorphic)
jit_chain_guard(:jne, jit, starting_context, asm, megamorphic_side_exit)
# If we don't have an instance variable index, then we need to
# transition out of the current shape.
if ivar_index.nil?
2023-03-02 23:44:57 -08:00
shape = C.rb_shape_get_shape_by_id(shape_id)
current_capacity = shape.capacity
new_capacity = current_capacity * 2
# If the object doesn't have the capacity to store the IV,
# then we'll need to allocate it.
needs_extension = shape.next_iv_index >= current_capacity
# We can write to the object, but we need to transition the shape
ivar_index = shape.next_iv_index
capa_shape =
if needs_extension
# We need to add an extended table to the object
# First, create an outgoing transition that increases the capacity
C.rb_shape_transition_shape_capa(shape, new_capacity)
else
nil
end
dest_shape =
if capa_shape
C.rb_shape_get_next(capa_shape, comptime_receiver, ivar_name)
else
C.rb_shape_get_next(shape, comptime_receiver, ivar_name)
end
new_shape_id = C.rb_shape_id(dest_shape)
if new_shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID
2023-03-02 23:44:57 -08:00
asm.incr_counter(:setivar_too_complex)
return CantCompile
end
if needs_extension
# Generate the C call so that runtime code will increase
# the capacity and set the buffer.
asm.mov(C_ARGS[0], :rax)
asm.mov(C_ARGS[1], current_capacity)
asm.mov(C_ARGS[2], new_capacity)
asm.call(C.rb_ensure_iv_list_size)
# Load the receiver again after the function call
asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)])
end
write_val = ctx.stack_pop(1)
jit_write_iv(asm, comptime_receiver, :rax, :rcx, ivar_index, write_val, needs_extension)
# Store the new shape
asm.comment('write shape')
asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # reload after jit_write_iv
asm.mov(DwordPtr[:rax, C.rb_shape_id_offset], new_shape_id)
2023-03-02 21:14:26 -08:00
else
# If the iv index already exists, then we don't need to
# transition to a new shape. The reason is because we find
# the iv index by searching up the shape tree. If we've
# made the transition already, then there's no reason to
# update the shape on the object. Just set the IV.
write_val = ctx.stack_pop(1)
2023-03-02 23:44:57 -08:00
jit_write_iv(asm, comptime_receiver, :rax, :rcx, ivar_index, write_val, false)
2023-03-02 21:14:26 -08:00
end
skip_wb = asm.new_label('skip_wb')
# If the value we're writing is an immediate, we don't need to WB
asm.test(write_val, C::RUBY_IMMEDIATE_MASK)
2023-03-02 21:14:26 -08:00
asm.jnz(skip_wb)
# If the value we're writing is nil or false, we don't need to WB
asm.cmp(write_val, Qnil)
asm.jbe(skip_wb)
asm.comment('write barrier')
2023-03-02 23:44:57 -08:00
asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)]) # reload after jit_write_iv
2023-03-02 21:14:26 -08:00
asm.mov(C_ARGS[1], write_val)
asm.call(C.rb_gc_writebarrier)
asm.write_label(skip_wb)
end
2023-02-08 17:27:21 -08:00
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 21:41:35 -08:00
def getclassvariable(jit, ctx, asm)
# rb_vm_getclassvariable can raise exceptions.
jit_prepare_routine_call(jit, ctx, asm)
asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:iseq)])
asm.mov(C_ARGS[1], CFP)
asm.mov(C_ARGS[2], jit.operand(0))
asm.mov(C_ARGS[3], jit.operand(1))
asm.call(C.rb_vm_getclassvariable)
top = ctx.stack_push
asm.mov(top, C_RET)
KeepCompiling
end
2023-03-18 21:49:42 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def setclassvariable(jit, ctx, asm)
# rb_vm_setclassvariable can raise exceptions.
jit_prepare_routine_call(jit, ctx, asm)
asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:iseq)])
asm.mov(C_ARGS[1], CFP)
asm.mov(C_ARGS[2], jit.operand(0))
asm.mov(C_ARGS[3], ctx.stack_pop(1))
asm.mov(C_ARGS[4], jit.operand(1))
asm.call(C.rb_vm_setclassvariable)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_getconstant_path(jit, ctx, asm)
# Cut the block for invalidation
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
ic = C.iseq_inline_constant_cache.new(jit.operand(0))
idlist = ic.segments
# Make sure there is an exit for this block as the interpreter might want
2023-03-06 23:17:25 -08:00
# to invalidate this block from rb_rjit_constant_ic_update().
# For now, we always take an entry exit even if it was a side exit.
Invariants.ensure_block_entry_exit(jit, cause: 'opt_getconstant_path')
# See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update().
ice = ic.entry
if ice.nil?
# In this case, leave a block that unconditionally side exits
# for the interpreter to invalidate.
asm.incr_counter(:optgetconst_not_cached)
return CantCompile
end
if ice.ic_cref # with cref
2023-03-03 22:14:15 -08:00
# Cache is keyed on a certain lexical scope. Use the interpreter's cache.
side_exit = side_exit(jit, ctx)
# Call function to verify the cache. It doesn't allocate or call methods.
asm.mov(C_ARGS[0], ic.to_i)
asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:ep)])
asm.call(C.rb_vm_ic_hit_p)
# Check the result. SysV only specifies one byte for _Bool return values,
# so it's important we only check one bit to ignore the higher bits in the register.
asm.test(C_RET, 1)
asm.jz(counted_exit(side_exit, :optgetconst_cache_miss))
asm.mov(:rax, ic.to_i) # inline_cache
asm.mov(:rax, [:rax, C.iseq_inline_constant_cache.offsetof(:entry)]) # ic_entry
asm.mov(:rax, [:rax, C.iseq_inline_constant_cache_entry.offsetof(:value)]) # ic_entry_val
# Push ic->entry->value
stack_top = ctx.stack_push
asm.mov(stack_top, :rax)
else # without cref
# TODO: implement this
# Optimize for single ractor mode.
# if !assume_single_ractor_mode(jit, ocb)
# return CantCompile
# end
# Invalidate output code on any constant writes associated with
# constants referenced within the current block.
Invariants.assume_stable_constant_names(jit, idlist)
putobject(jit, ctx, asm, val: ice.value)
end
jump_to_next_insn(jit, ctx, asm)
EndBlock
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 22:49:31 -08:00
def getconstant(jit, ctx, asm)
id = jit.operand(0)
# vm_get_ev_const can raise exceptions.
jit_prepare_routine_call(jit, ctx, asm)
allow_nil_opnd = ctx.stack_pop(1)
klass_opnd = ctx.stack_pop(1)
asm.mov(C_ARGS[0], EC)
asm.mov(C_ARGS[1], klass_opnd)
asm.mov(C_ARGS[2], id)
asm.mov(C_ARGS[3], allow_nil_opnd)
asm.call(C.rb_vm_get_ev_const)
top = ctx.stack_push
asm.mov(top, C_RET)
KeepCompiling
end
2022-12-28 13:50:24 -08:00
# setconstant
2023-03-18 21:20:37 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def getglobal(jit, ctx, asm)
gid = jit.operand(0)
# Save the PC and SP because we might make a Ruby call for warning
jit_prepare_routine_call(jit, ctx, asm)
asm.mov(C_ARGS[0], gid)
asm.call(C.rb_gvar_get)
top = ctx.stack_push
asm.mov(top, C_RET)
KeepCompiling
end
2022-12-28 13:50:24 -08:00
# setglobal
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2022-12-26 14:09:45 -08:00
def putnil(jit, ctx, asm)
2023-02-08 09:30:47 -08:00
putobject(jit, ctx, asm, val: Qnil)
2022-12-17 13:39:35 -08:00
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-01-07 13:39:33 -08:00
def putself(jit, ctx, asm)
2023-02-08 09:30:47 -08:00
stack_top = ctx.stack_push
2023-01-07 13:39:33 -08:00
asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)])
2023-02-08 09:30:47 -08:00
asm.mov(stack_top, :rax)
2023-01-07 13:39:33 -08:00
KeepCompiling
end
2022-12-28 14:43:04 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-01-07 13:54:26 -08:00
def putobject(jit, ctx, asm, val: jit.operand(0))
2022-12-28 14:43:04 -08:00
# Push it to the stack
2023-02-08 09:30:47 -08:00
stack_top = ctx.stack_push
2022-12-28 14:43:04 -08:00
if asm.imm32?(val)
2023-02-08 09:30:47 -08:00
asm.mov(stack_top, val)
2022-12-28 14:43:04 -08:00
else # 64-bit immediates can't be directly written to memory
asm.mov(:rax, val)
2023-02-08 09:30:47 -08:00
asm.mov(stack_top, :rax)
2022-12-28 14:43:04 -08:00
end
2023-02-08 09:30:47 -08:00
# TODO: GC offsets?
2022-12-28 14:43:04 -08:00
KeepCompiling
end
2023-03-17 23:34:36 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def putspecialobject(jit, ctx, asm)
object_type = jit.operand(0)
if object_type == C::VM_SPECIAL_OBJECT_VMCORE
stack_top = ctx.stack_push
asm.mov(:rax, C.rb_mRubyVMFrozenCore)
asm.mov(stack_top, :rax)
KeepCompiling
else
# TODO: implement for VM_SPECIAL_OBJECT_CBASE and
# VM_SPECIAL_OBJECT_CONST_BASE
CantCompile
end
end
2023-02-21 00:19:37 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-21 00:19:37 -08:00
def putstring(jit, ctx, asm)
put_val = jit.operand(0, ruby: true)
# Save the PC and SP because the callee will allocate
jit_prepare_routine_call(jit, ctx, asm)
asm.mov(C_ARGS[0], EC)
asm.mov(C_ARGS[1], to_value(put_val))
asm.call(C.rb_ec_str_resurrect)
stack_top = ctx.stack_push
asm.mov(stack_top, C_RET)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 22:21:56 -08:00
def concatstrings(jit, ctx, asm)
n = jit.operand(0)
# Save the PC and SP because we are allocating
jit_prepare_routine_call(jit, ctx, asm)
asm.lea(:rax, ctx.sp_opnd(-C.VALUE.size * n))
# call rb_str_concat_literals(size_t n, const VALUE *strings);
asm.mov(C_ARGS[0], n)
asm.mov(C_ARGS[1], :rax)
asm.call(C.rb_str_concat_literals)
ctx.stack_pop(n)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2023-03-03 22:17:34 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 22:17:34 -08:00
def anytostring(jit, ctx, asm)
# Save the PC and SP since we might call #to_s
jit_prepare_routine_call(jit, ctx, asm)
str = ctx.stack_pop(1)
val = ctx.stack_pop(1)
asm.mov(C_ARGS[0], str)
asm.mov(C_ARGS[1], val)
asm.call(C.rb_obj_as_string_result)
# Push the return value
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2023-03-18 21:37:16 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def toregexp(jit, ctx, asm)
opt = jit.operand(0, signed: true)
cnt = jit.operand(1)
# Save the PC and SP because this allocates an object and could
# raise an exception.
jit_prepare_routine_call(jit, ctx, asm)
asm.lea(:rax, ctx.sp_opnd(-C.VALUE.size * cnt)) # values_ptr
ctx.stack_pop(cnt)
asm.mov(C_ARGS[0], 0)
asm.mov(C_ARGS[1], cnt)
asm.mov(C_ARGS[2], :rax) # values_ptr
asm.call(C.rb_ary_tmp_new_from_values)
# Save the array so we can clear it later
asm.push(C_RET)
asm.push(C_RET) # Alignment
2023-03-18 21:39:24 -07:00
asm.mov(C_ARGS[0], C_RET)
2023-03-18 21:37:16 -07:00
asm.mov(C_ARGS[1], opt)
asm.call(C.rb_reg_new_ary)
# The actual regex is in RAX now. Pop the temp array from
# rb_ary_tmp_new_from_values into C arg regs so we can clear it
asm.pop(:rcx) # Alignment
asm.pop(:rcx) # ary
# The value we want to push on the stack is in RAX right now
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
# Clear the temp array.
asm.mov(C_ARGS[0], :rcx) # ary
asm.call(C.rb_ary_clear)
KeepCompiling
end
2023-03-18 21:42:10 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def intern(jit, ctx, asm)
# Save the PC and SP because we might allocate
jit_prepare_routine_call(jit, ctx, asm);
str = ctx.stack_pop(1)
asm.mov(C_ARGS[0], str)
asm.call(C.rb_str_intern)
# Push the return value
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2023-02-21 00:02:56 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-21 00:02:56 -08:00
def newarray(jit, ctx, asm)
n = jit.operand(0)
# Save the PC and SP because we are allocating
jit_prepare_routine_call(jit, ctx, asm)
# If n is 0, then elts is never going to be read, so we can just pass null
if n == 0
values_ptr = 0
else
asm.comment('load pointer to array elts')
offset_magnitude = C.VALUE.size * n
values_opnd = ctx.sp_opnd(-(offset_magnitude))
asm.lea(:rax, values_opnd)
values_ptr = :rax
end
# call rb_ec_ary_new_from_values(struct rb_execution_context_struct *ec, long n, const VALUE *elts);
asm.mov(C_ARGS[0], EC)
asm.mov(C_ARGS[1], n)
asm.mov(C_ARGS[2], values_ptr)
asm.call(C.rb_ec_ary_new_from_values)
ctx.stack_pop(n)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2022-12-28 13:50:24 -08:00
# newarraykwsplat
2023-02-15 21:26:04 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-15 21:26:04 -08:00
def duparray(jit, ctx, asm)
ary = jit.operand(0)
# Save the PC and SP because we are allocating
jit_prepare_routine_call(jit, ctx, asm)
# call rb_ary_resurrect(VALUE ary);
asm.comment('call rb_ary_resurrect')
asm.mov(C_ARGS[0], ary)
asm.call(C.rb_ary_resurrect)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2022-12-28 13:50:24 -08:00
# duphash
2023-02-15 21:26:04 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-15 21:26:04 -08:00
def expandarray(jit, ctx, asm)
# Both arguments are rb_num_t which is unsigned
num = jit.operand(0)
flag = jit.operand(1)
# If this instruction has the splat flag, then bail out.
if flag & 0x01 != 0
asm.incr_counter(:expandarray_splat)
return CantCompile
end
# If this instruction has the postarg flag, then bail out.
if flag & 0x02 != 0
asm.incr_counter(:expandarray_postarg)
return CantCompile
end
side_exit = side_exit(jit, ctx)
array_opnd = ctx.stack_pop(1)
# num is the number of requested values. If there aren't enough in the
# array then we're going to push on nils.
# TODO: implement this
# Move the array from the stack and check that it's an array.
asm.mov(:rax, array_opnd)
guard_object_is_heap(asm, :rax, counted_exit(side_exit, :expandarray_not_array))
guard_object_is_array(asm, :rax, :rcx, counted_exit(side_exit, :expandarray_not_array))
# If we don't actually want any values, then just return.
if num == 0
return KeepCompiling
end
jit_array_len(asm, :rax, :rcx)
# Only handle the case where the number of values in the array is greater
# than or equal to the number of values requested.
asm.cmp(:rcx, num)
asm.jl(counted_exit(side_exit, :expandarray_rhs_too_small))
# Conditionally load the address of the heap array into REG1.
# (struct RArray *)(obj)->as.heap.ptr
#asm.mov(:rax, array_opnd)
asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)])
asm.test(:rcx, C::RARRAY_EMBED_FLAG);
2023-02-15 21:26:04 -08:00
asm.mov(:rcx, [:rax, C.RArray.offsetof(:as, :heap, :ptr)])
# Load the address of the embedded array into REG1.
# (struct RArray *)(obj)->as.ary
asm.lea(:rax, [:rax, C.RArray.offsetof(:as, :ary)])
asm.cmovnz(:rcx, :rax)
# Loop backward through the array and push each element onto the stack.
(num - 1).downto(0).each do |i|
top = ctx.stack_push
asm.mov(:rax, [:rcx, i * C.VALUE.size])
asm.mov(top, :rax)
end
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 22:54:24 -08:00
def concatarray(jit, ctx, asm)
# Save the PC and SP because the callee may allocate
# Note that this modifies REG_SP, which is why we do it first
jit_prepare_routine_call(jit, ctx, asm)
# Get the operands from the stack
ary2st_opnd = ctx.stack_pop(1)
ary1_opnd = ctx.stack_pop(1)
# Call rb_vm_concat_array(ary1, ary2st)
asm.mov(C_ARGS[0], ary1_opnd)
asm.mov(C_ARGS[1], ary2st_opnd)
asm.call(C.rb_vm_concat_array)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2023-02-21 00:16:05 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-21 00:16:05 -08:00
def splatarray(jit, ctx, asm)
flag = jit.operand(0)
# Save the PC and SP because the callee may allocate
# Note that this modifies REG_SP, which is why we do it first
jit_prepare_routine_call(jit, ctx, asm)
# Get the operands from the stack
ary_opnd = ctx.stack_pop(1)
# Call rb_vm_splat_array(flag, ary)
asm.mov(C_ARGS[0], flag)
asm.mov(C_ARGS[1], ary_opnd)
asm.call(C.rb_vm_splat_array)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-21 00:32:37 -08:00
def newhash(jit, ctx, asm)
num = jit.operand(0)
# Save the PC and SP because we are allocating
jit_prepare_routine_call(jit, ctx, asm)
if num != 0
# val = rb_hash_new_with_size(num / 2);
asm.mov(C_ARGS[0], num / 2)
asm.call(C.rb_hash_new_with_size)
# Save the allocated hash as we want to push it after insertion
asm.push(C_RET)
asm.push(C_RET) # x86 alignment
# Get a pointer to the values to insert into the hash
asm.lea(:rcx, ctx.stack_opnd(num - 1))
# rb_hash_bulk_insert(num, STACK_ADDR_FROM_TOP(num), val);
asm.mov(C_ARGS[0], num)
asm.mov(C_ARGS[1], :rcx)
asm.mov(C_ARGS[2], C_RET)
asm.call(C.rb_hash_bulk_insert)
asm.pop(:rax)
asm.pop(:rax)
ctx.stack_pop(num)
stack_ret = ctx.stack_push
asm.mov(stack_ret, :rax)
else
# val = rb_hash_new();
asm.call(C.rb_hash_new)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
end
KeepCompiling
end
2023-03-18 21:24:28 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def newrange(jit, ctx, asm)
flag = jit.operand(0)
# rb_range_new() allocates and can raise
jit_prepare_routine_call(jit, ctx, asm)
# val = rb_range_new(low, high, (int)flag);
asm.mov(C_ARGS[0], ctx.stack_opnd(1))
asm.mov(C_ARGS[1], ctx.stack_opnd(0))
asm.mov(C_ARGS[2], flag)
asm.call(C.rb_range_new)
ctx.stack_pop(2)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2023-02-08 17:33:27 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 17:33:27 -08:00
def pop(jit, ctx, asm)
ctx.stack_pop
KeepCompiling
end
2023-02-08 17:17:36 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 17:17:36 -08:00
def dup(jit, ctx, asm)
val1 = ctx.stack_opnd(0)
val2 = ctx.stack_push
asm.mov(:rax, val1)
asm.mov(val2, :rax)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-14 22:52:12 -08:00
def dupn(jit, ctx, asm)
n = jit.operand(0)
# In practice, seems to be only used for n==2
if n != 2
return CantCompile
end
opnd1 = ctx.stack_opnd(1)
opnd0 = ctx.stack_opnd(0)
dst1 = ctx.stack_push
asm.mov(:rax, opnd1)
asm.mov(dst1, :rax)
dst0 = ctx.stack_push
asm.mov(:rax, opnd0)
asm.mov(dst0, :rax)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-21 00:22:50 -08:00
def swap(jit, ctx, asm)
stack0_mem = ctx.stack_opnd(0)
stack1_mem = ctx.stack_opnd(1)
asm.mov(:rax, stack0_mem)
asm.mov(:rcx, stack1_mem)
asm.mov(stack0_mem, :rcx)
asm.mov(stack1_mem, :rax)
KeepCompiling
end
2022-12-28 13:50:24 -08:00
# opt_reverse
2023-02-21 00:11:56 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-21 00:11:56 -08:00
def topn(jit, ctx, asm)
n = jit.operand(0)
top_n_val = ctx.stack_opnd(n)
loc0 = ctx.stack_push
asm.mov(:rax, top_n_val)
asm.mov(loc0, :rax)
KeepCompiling
end
2023-02-14 22:52:12 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-14 22:52:12 -08:00
def setn(jit, ctx, asm)
n = jit.operand(0)
top_val = ctx.stack_pop(0)
dst_opnd = ctx.stack_opnd(n)
asm.mov(:rax, top_val)
asm.mov(dst_opnd, :rax)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-20 23:57:35 -08:00
def adjuststack(jit, ctx, asm)
n = jit.operand(0)
ctx.stack_pop(n)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-28 22:45:59 -08:00
def defined(jit, ctx, asm)
op_type = jit.operand(0)
obj = jit.operand(1, ruby: true)
pushval = jit.operand(2, ruby: true)
# Save the PC and SP because the callee may allocate
# Note that this modifies REG_SP, which is why we do it first
jit_prepare_routine_call(jit, ctx, asm)
# Get the operands from the stack
v_opnd = ctx.stack_pop(1)
# Call vm_defined(ec, reg_cfp, op_type, obj, v)
asm.mov(C_ARGS[0], EC)
asm.mov(C_ARGS[1], CFP)
asm.mov(C_ARGS[2], op_type)
asm.mov(C_ARGS[3], to_value(obj))
asm.mov(C_ARGS[4], v_opnd)
asm.call(C.rb_vm_defined)
asm.test(C_RET, 255)
asm.mov(:rax, Qnil)
asm.mov(:rcx, to_value(pushval))
asm.cmovnz(:rax, :rcx)
# Push the return value onto the stack
stack_ret = ctx.stack_push
asm.mov(stack_ret, :rax)
KeepCompiling
end
2023-03-12 22:07:02 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def definedivar(jit, ctx, asm)
# Defer compilation so we can specialize base on a runtime receiver
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
2023-03-12 22:07:02 -07:00
ivar_name = jit.operand(0)
# Value that will be pushed on the stack if the ivar is defined. In practice this is always the
# string "instance-variable". If the ivar is not defined, nil will be pushed instead.
2023-03-12 22:07:02 -07:00
pushval = jit.operand(2)
# Get the receiver
recv = :rcx
asm.mov(recv, [CFP, C.rb_control_frame_t.offsetof(:self)])
2023-03-12 22:07:02 -07:00
# Specialize base on compile time values
comptime_receiver = jit.peek_at_self
2023-03-12 22:07:02 -07:00
if shape_too_complex?(comptime_receiver)
# Fall back to calling rb_ivar_defined
2023-03-12 22:07:02 -07:00
# Save the PC and SP because the callee may allocate
# Note that this modifies REG_SP, which is why we do it first
jit_prepare_routine_call(jit, ctx, asm) # clobbers :rax
# Call rb_ivar_defined(recv, ivar_name)
asm.mov(C_ARGS[0], recv)
asm.mov(C_ARGS[1], ivar_name)
asm.call(C.rb_ivar_defined)
# if (rb_ivar_defined(recv, ivar_name)) {
# val = pushval;
# }
asm.test(C_RET, 255)
asm.mov(:rax, Qnil)
asm.mov(:rcx, pushval)
asm.cmovnz(:rax, :rcx)
# Push the return value onto the stack
stack_ret = ctx.stack_push
asm.mov(stack_ret, :rax)
2023-03-12 22:07:02 -07:00
return KeepCompiling
end
2023-03-12 22:07:02 -07:00
shape_id = C.rb_shape_get_shape_id(comptime_receiver)
ivar_exists = C.rb_shape_get_iv_index(shape_id, ivar_name)
side_exit = side_exit(jit, ctx)
# Guard heap object (recv_opnd must be used before stack_pop)
guard_object_is_heap(asm, recv, side_exit)
shape_opnd = DwordPtr[recv, C.rb_shape_id_offset]
asm.comment('guard shape')
asm.cmp(shape_opnd, shape_id)
jit_chain_guard(:jne, jit, ctx, asm, side_exit)
result = ivar_exists ? pushval : Qnil
putobject(jit, ctx, asm, val: result)
# Jump to next instruction. This allows guard chains to share the same successor.
jump_to_next_insn(jit, ctx, asm)
return EndBlock
2023-03-12 22:07:02 -07:00
end
2022-12-28 13:50:24 -08:00
# checkmatch
2023-03-18 21:14:35 -07:00
2023-03-18 21:49:42 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-18 21:14:35 -07:00
def checkkeyword(jit, ctx, asm)
# When a keyword is unspecified past index 32, a hash will be used
# instead. This can only happen in iseqs taking more than 32 keywords.
if jit.iseq.body.param.keyword.num >= 32
return CantCompile
end
# The EP offset to the undefined bits local
bits_offset = jit.operand(0)
# The index of the keyword we want to check
index = jit.operand(1, signed: true)
# Load environment pointer EP
ep_reg = :rax
jit_get_ep(asm, 0, reg: ep_reg)
# VALUE kw_bits = *(ep - bits)
bits_opnd = [ep_reg, C.VALUE.size * -bits_offset]
# unsigned int b = (unsigned int)FIX2ULONG(kw_bits);
# if ((b & (0x01 << idx))) {
#
# We can skip the FIX2ULONG conversion by shifting the bit we test
bit_test = 0x01 << (index + 1)
asm.test(bits_opnd, bit_test)
asm.mov(:rax, Qfalse)
asm.mov(:rcx, Qtrue)
asm.cmovz(:rax, :rcx)
stack_ret = ctx.stack_push
asm.mov(stack_ret, :rax)
KeepCompiling
end
2022-12-28 13:50:24 -08:00
# checktype
# defineclass
# definemethod
# definesmethod
2023-02-28 22:50:04 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-28 22:50:04 -08:00
def send(jit, ctx, asm)
# Specialize on a compile-time receiver, and split a block for chain guards
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
cd = C.rb_call_data.new(jit.operand(0))
blockiseq = jit.operand(1)
2023-02-28 23:22:12 -08:00
block_handler = jit_caller_setup_arg_block(jit, ctx, asm, cd.ci, blockiseq, false)
if block_handler == CantCompile
2023-02-28 22:50:04 -08:00
return CantCompile
end
# calling->ci
mid = C.vm_ci_mid(cd.ci)
calling = build_calling(ci: cd.ci, block_handler:)
2023-02-28 22:50:04 -08:00
# vm_sendish
cme, comptime_recv_klass = jit_search_method(jit, ctx, asm, mid, calling)
2023-02-28 22:50:04 -08:00
if cme == CantCompile
return CantCompile
end
jit_call_general(jit, ctx, asm, mid, calling, cme, comptime_recv_klass)
2023-02-28 22:50:04 -08:00
end
2023-01-07 21:24:30 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-24 14:48:02 -08:00
def opt_send_without_block(jit, ctx, asm, cd: C.rb_call_data.new(jit.operand(0)))
# Specialize on a compile-time receiver, and split a block for chain guards
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
# calling->ci
mid = C.vm_ci_mid(cd.ci)
calling = build_calling(ci: cd.ci, block_handler: C::VM_BLOCK_HANDLER_NONE)
2023-02-24 14:48:02 -08:00
# vm_sendish
cme, comptime_recv_klass = jit_search_method(jit, ctx, asm, mid, calling)
2023-02-24 14:48:02 -08:00
if cme == CantCompile
return CantCompile
end
jit_call_general(jit, ctx, asm, mid, calling, cme, comptime_recv_klass)
2023-01-07 21:24:30 -08:00
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 22:03:23 -08:00
def objtostring(jit, ctx, asm)
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
recv = ctx.stack_opnd(0)
comptime_recv = jit.peek_at_stack(0)
2023-03-18 22:57:31 -07:00
if C.RB_TYPE_P(comptime_recv, C::RUBY_T_STRING)
2023-03-03 22:03:23 -08:00
side_exit = side_exit(jit, ctx)
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, comptime_recv, side_exit)
# No work needed. The string value is already on the top of the stack.
KeepCompiling
else
cd = C.rb_call_data.new(jit.operand(0))
opt_send_without_block(jit, ctx, asm, cd:)
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 22:45:25 -08:00
def opt_str_freeze(jit, ctx, asm)
unless Invariants.assume_bop_not_redefined(jit, C::STRING_REDEFINED_OP_FLAG, C::BOP_FREEZE)
2023-03-03 22:45:25 -08:00
return CantCompile;
end
str = jit.operand(0, ruby: true)
# Push the return value onto the stack
stack_ret = ctx.stack_push
asm.mov(:rax, to_value(str))
asm.mov(stack_ret, :rax)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_nil_p(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2022-12-28 13:50:24 -08:00
# opt_str_uminus
# opt_newarray_max
2023-03-01 23:38:12 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-01 23:38:12 -08:00
def opt_newarray_min(jit, ctx, asm)
num = jit.operand(0)
# Save the PC and SP because we may allocate
jit_prepare_routine_call(jit, ctx, asm)
offset_magnitude = C.VALUE.size * num
values_opnd = ctx.sp_opnd(-offset_magnitude)
asm.lea(:rax, values_opnd)
asm.mov(C_ARGS[0], EC)
asm.mov(C_ARGS[1], num)
asm.mov(C_ARGS[2], :rax)
asm.call(C.rb_vm_opt_newarray_min)
ctx.stack_pop(num)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2023-02-24 14:48:02 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-24 14:48:02 -08:00
def invokesuper(jit, ctx, asm)
# Specialize on a compile-time receiver, and split a block for chain guards
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
cd = C.rb_call_data.new(jit.operand(0))
blockiseq = jit.operand(1)
2023-02-28 23:22:12 -08:00
block_handler = jit_caller_setup_arg_block(jit, ctx, asm, cd.ci, blockiseq, true)
if block_handler == CantCompile
2023-02-28 22:50:04 -08:00
return CantCompile
end
2023-02-24 14:48:02 -08:00
# calling->ci
mid = C.vm_ci_mid(cd.ci)
calling = build_calling(ci: cd.ci, block_handler:)
2023-02-24 14:48:02 -08:00
# vm_sendish
cme = jit_search_super_method(jit, ctx, asm, mid, calling)
2023-02-24 14:48:02 -08:00
if cme == CantCompile
return CantCompile
end
jit_call_general(jit, ctx, asm, mid, calling, cme, nil)
2023-02-24 14:48:02 -08:00
end
2023-03-12 22:27:43 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def invokeblock(jit, ctx, asm)
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
# Get call info
cd = C.rb_call_data.new(jit.operand(0))
calling = build_calling(ci: cd.ci, block_handler: :captured)
2023-03-12 22:27:43 -07:00
# Get block_handler
cfp = jit.cfp
lep = C.rb_vm_ep_local_ep(cfp.ep)
comptime_handler = lep[C::VM_ENV_DATA_INDEX_SPECVAL]
# Handle each block_handler type
if comptime_handler == C::VM_BLOCK_HANDLER_NONE # no block given
asm.incr_counter(:invokeblock_none)
CantCompile
elsif comptime_handler & 0x3 == 0x1 # VM_BH_ISEQ_BLOCK_P
2023-03-12 22:27:43 -07:00
asm.comment('get local EP')
ep_reg = :rax
jit_get_lep(jit, asm, reg: ep_reg)
asm.mov(:rax, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler_opnd
asm.comment('guard block_handler type')
side_exit = side_exit(jit, ctx)
asm.mov(:rcx, :rax)
asm.and(:rcx, 0x3) # block_handler is a tagged pointer
asm.cmp(:rcx, 0x1) # VM_BH_ISEQ_BLOCK_P
tag_changed_exit = counted_exit(side_exit, :invokeblock_tag_changed)
jit_chain_guard(:jne, jit, ctx, asm, tag_changed_exit)
comptime_captured = C.rb_captured_block.new(comptime_handler & ~0x3)
comptime_iseq = comptime_captured.code.iseq
asm.comment('guard known ISEQ')
asm.and(:rax, ~0x3) # captured
asm.mov(:rax, [:rax, C.VALUE.size * 2]) # captured->iseq
asm.mov(:rcx, comptime_iseq.to_i)
asm.cmp(:rax, :rcx)
block_changed_exit = counted_exit(side_exit, :invokeblock_iseq_block_changed)
jit_chain_guard(:jne, jit, ctx, asm, block_changed_exit)
jit_call_iseq(jit, ctx, asm, nil, calling, comptime_iseq, frame_type: C::VM_FRAME_MAGIC_BLOCK)
2023-03-12 22:27:43 -07:00
elsif comptime_handler & 0x3 == 0x3 # VM_BH_IFUNC_P
2023-03-19 23:19:58 -07:00
# We aren't handling CALLER_SETUP_ARG and CALLER_REMOVE_EMPTY_KW_SPLAT yet.
if calling.flags & C::VM_CALL_ARGS_SPLAT != 0
asm.incr_counter(:invokeblock_ifunc_args_splat)
return CantCompile
end
if calling.flags & C::VM_CALL_KW_SPLAT != 0
asm.incr_counter(:invokeblock_ifunc_kw_splat)
return CantCompile
end
asm.comment('get local EP')
jit_get_lep(jit, asm, reg: :rax)
asm.mov(:rcx, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler_opnd
asm.comment('guard block_handler type');
side_exit = side_exit(jit, ctx)
asm.mov(:rax, :rcx) # block_handler_opnd
asm.and(:rax, 0x3) # tag_opnd: block_handler is a tagged pointer
asm.cmp(:rax, 0x3) # VM_BH_IFUNC_P
tag_changed_exit = counted_exit(side_exit, :invokeblock_tag_changed)
jit_chain_guard(:jne, jit, ctx, asm, tag_changed_exit)
# The cfunc may not be leaf
jit_prepare_routine_call(jit, ctx, asm) # clobbers :rax
asm.comment('call ifunc')
asm.and(:rcx, ~0x3) # captured_opnd
asm.lea(:rax, ctx.sp_opnd(-calling.argc * C.VALUE.size)) # argv
asm.mov(C_ARGS[0], EC)
asm.mov(C_ARGS[1], :rcx) # captured_opnd
asm.mov(C_ARGS[2], calling.argc)
asm.mov(C_ARGS[3], :rax) # argv
asm.call(C.rb_vm_yield_with_cfunc)
ctx.stack_pop(calling.argc)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
# Share the successor with other chains
jump_to_next_insn(jit, ctx, asm)
EndBlock
2023-03-12 22:27:43 -07:00
elsif symbol?(comptime_handler)
asm.incr_counter(:invokeblock_symbol)
CantCompile
else # Proc
asm.incr_counter(:invokeblock_proc)
CantCompile
end
end
2022-12-28 13:50:24 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2022-12-26 14:09:45 -08:00
def leave(jit, ctx, asm)
2023-02-03 22:42:13 -08:00
assert_equal(ctx.stack_size, 1)
2022-12-26 14:09:45 -08:00
2023-02-07 14:42:58 -08:00
jit_check_ints(jit, ctx, asm)
2022-12-23 14:17:32 -08:00
2022-12-28 13:16:02 -08:00
asm.comment('pop stack frame')
2023-01-07 21:24:30 -08:00
asm.lea(:rax, [CFP, C.rb_control_frame_t.size])
asm.mov(CFP, :rax)
asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], :rax)
2022-12-17 13:39:35 -08:00
2023-01-07 21:24:30 -08:00
# Return a value (for compile_leave_exit)
2023-02-06 15:44:34 -08:00
ret_opnd = ctx.stack_pop
asm.mov(:rax, ret_opnd)
2022-12-23 14:46:39 -08:00
2023-01-07 21:24:30 -08:00
# Set caller's SP and push a value to its stack (for JIT)
2023-02-03 22:42:13 -08:00
asm.mov(SP, [CFP, C.rb_control_frame_t.offsetof(:sp)]) # Note: SP is in the position after popping a receiver and arguments
2023-01-07 21:24:30 -08:00
asm.mov([SP], :rax)
# Jump to cfp->jit_return
asm.jmp([CFP, -C.rb_control_frame_t.size + C.rb_control_frame_t.offsetof(:jit_return)])
2022-12-23 14:46:39 -08:00
2022-12-17 13:39:35 -08:00
EndBlock
end
2022-12-23 14:17:32 -08:00
2023-03-17 23:27:16 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def throw(jit, ctx, asm)
throw_state = jit.operand(0)
asm.mov(:rcx, ctx.stack_pop(1)) # throwobj
# THROW_DATA_NEW allocates. Save SP for GC and PC for allocation tracing as
# well as handling the catch table. However, not using jit_prepare_routine_call
# since we don't need a patch point for this implementation.
jit_save_pc(jit, asm) # clobbers rax
jit_save_sp(ctx, asm)
# rb_vm_throw verifies it's a valid throw, sets ec->tag->state, and returns throw
# data, which is throwobj or a vm_throw_data wrapping it. When ec->tag->state is
# set, JIT code callers will handle the throw with vm_exec_handle_exception.
asm.mov(C_ARGS[0], EC)
asm.mov(C_ARGS[1], CFP)
asm.mov(C_ARGS[2], throw_state)
# asm.mov(C_ARGS[3], :rcx) # same reg
asm.call(C.rb_vm_throw)
asm.comment('exit from throw')
asm.pop(SP)
asm.pop(EC)
asm.pop(CFP)
# return C_RET as C_RET
asm.ret
EndBlock
end
2023-02-08 17:42:16 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 17:42:16 -08:00
def jump(jit, ctx, asm)
# Check for interrupts, but only on backward branches that may create loops
2023-02-13 07:30:25 -08:00
jump_offset = jit.operand(0, signed: true)
if jump_offset < 0
jit_check_ints(jit, ctx, asm)
end
2023-02-08 17:42:16 -08:00
pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)
jit_direct_jump(jit.iseq, pc, ctx, asm)
2023-02-08 17:42:16 -08:00
EndBlock
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 07:30:25 -08:00
def branchif(jit, ctx, asm)
# Check for interrupts, but only on backward branches that may create loops
jump_offset = jit.operand(0, signed: true)
if jump_offset < 0
jit_check_ints(jit, ctx, asm)
end
# TODO: skip check for known truthy
# This `test` sets ZF only for Qnil and Qfalse, which let jz jump.
val = ctx.stack_pop
asm.test(val, ~Qnil)
# Set stubs
branch_stub = BranchStub.new(
iseq: jit.iseq,
shape: Default,
target0: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)), # branch target
target1: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * jit.insn.len), # fallthrough
)
branch_stub.target0.address = Assembler.new.then do |ocb_asm|
@exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true)
@ocb.write(ocb_asm)
end
branch_stub.target1.address = Assembler.new.then do |ocb_asm|
@exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false)
@ocb.write(ocb_asm)
end
# Jump to target0 on jnz
branch_stub.compile = proc do |branch_asm|
branch_asm.comment("branchif #{branch_stub.shape}")
branch_asm.stub(branch_stub) do
case branch_stub.shape
in Default
branch_asm.jnz(branch_stub.target0.address)
branch_asm.jmp(branch_stub.target1.address)
in Next0
branch_asm.jz(branch_stub.target1.address)
in Next1
branch_asm.jnz(branch_stub.target0.address)
end
end
end
branch_stub.compile.call(asm)
EndBlock
end
2023-01-07 13:21:14 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-01-07 13:21:14 -08:00
def branchunless(jit, ctx, asm)
# Check for interrupts, but only on backward branches that may create loops
2023-02-13 07:30:25 -08:00
jump_offset = jit.operand(0, signed: true)
if jump_offset < 0
jit_check_ints(jit, ctx, asm)
end
2023-01-07 13:21:14 -08:00
# TODO: skip check for known truthy
# This `test` sets ZF only for Qnil and Qfalse, which let jz jump.
2023-02-08 17:37:25 -08:00
val = ctx.stack_pop
asm.test(val, ~Qnil)
2023-01-07 13:21:14 -08:00
# Set stubs
branch_stub = BranchStub.new(
iseq: jit.iseq,
2023-02-07 00:17:13 -08:00
shape: Default,
target0: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)), # branch target
2023-02-09 16:25:06 -08:00
target1: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * jit.insn.len), # fallthrough
2023-01-07 13:21:14 -08:00
)
2023-02-07 00:17:13 -08:00
branch_stub.target0.address = Assembler.new.then do |ocb_asm|
2023-02-07 11:36:45 -08:00
@exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true)
2023-01-07 13:21:14 -08:00
@ocb.write(ocb_asm)
end
2023-02-07 00:17:13 -08:00
branch_stub.target1.address = Assembler.new.then do |ocb_asm|
2023-02-07 11:36:45 -08:00
@exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false)
2023-01-07 13:21:14 -08:00
@ocb.write(ocb_asm)
end
2023-02-07 00:17:13 -08:00
# Jump to target0 on jz
branch_stub.compile = proc do |branch_asm|
branch_asm.comment("branchunless #{branch_stub.shape}")
2023-01-07 13:21:14 -08:00
branch_asm.stub(branch_stub) do
2023-02-07 00:17:13 -08:00
case branch_stub.shape
in Default
branch_asm.jz(branch_stub.target0.address)
branch_asm.jmp(branch_stub.target1.address)
in Next0
branch_asm.jnz(branch_stub.target1.address)
in Next1
branch_asm.jz(branch_stub.target0.address)
end
2023-01-07 13:21:14 -08:00
end
end
2023-02-07 00:17:13 -08:00
branch_stub.compile.call(asm)
2023-01-07 13:21:14 -08:00
EndBlock
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 22:42:03 -08:00
def branchnil(jit, ctx, asm)
# Check for interrupts, but only on backward branches that may create loops
jump_offset = jit.operand(0, signed: true)
if jump_offset < 0
jit_check_ints(jit, ctx, asm)
end
# TODO: skip check for known truthy
val = ctx.stack_pop
asm.cmp(val, Qnil)
# Set stubs
branch_stub = BranchStub.new(
iseq: jit.iseq,
shape: Default,
target0: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)), # branch target
target1: BranchTarget.new(ctx:, pc: jit.pc + C.VALUE.size * jit.insn.len), # fallthrough
)
branch_stub.target0.address = Assembler.new.then do |ocb_asm|
@exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true)
@ocb.write(ocb_asm)
end
branch_stub.target1.address = Assembler.new.then do |ocb_asm|
@exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false)
@ocb.write(ocb_asm)
end
# Jump to target0 on je
branch_stub.compile = proc do |branch_asm|
branch_asm.comment("branchnil #{branch_stub.shape}")
branch_asm.stub(branch_stub) do
case branch_stub.shape
in Default
branch_asm.je(branch_stub.target0.address)
branch_asm.jmp(branch_stub.target1.address)
in Next0
branch_asm.jne(branch_stub.target1.address)
in Next1
branch_asm.je(branch_stub.target0.address)
end
end
end
branch_stub.compile.call(asm)
EndBlock
end
2022-12-28 13:50:24 -08:00
# once
2023-03-03 21:10:17 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 21:10:17 -08:00
def opt_case_dispatch(jit, ctx, asm)
# Normally this instruction would lookup the key in a hash and jump to an
# offset based on that.
# Instead we can take the fallback case and continue with the next
# instruction.
# We'd hope that our jitted code will be sufficiently fast without the
# hash lookup, at least for small hashes, but it's worth revisiting this
# assumption in the future.
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
starting_context = ctx.dup
case_hash = jit.operand(0, ruby: true)
else_offset = jit.operand(1)
# Try to reorder case/else branches so that ones that are actually used come first.
# Supporting only Fixnum for now so that the implementation can be an equality check.
key_opnd = ctx.stack_pop(1)
comptime_key = jit.peek_at_stack(0)
# Check that all cases are fixnums to avoid having to register BOP assumptions on
# all the types that case hashes support. This spends compile time to save memory.
if fixnum?(comptime_key) && comptime_key <= 2**32 && C.rb_hash_keys(case_hash).all? { |key| fixnum?(key) }
unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_EQQ)
return CantCompile
end
# Check if the key is the same value
asm.cmp(key_opnd, comptime_key)
side_exit = side_exit(jit, starting_context)
jit_chain_guard(:jne, jit, starting_context, asm, side_exit)
# Get the offset for the compile-time key
offset = C.rb_hash_stlike_lookup(case_hash, comptime_key)
# NOTE: If we hit the else branch with various values, it could negatively impact the performance.
jump_offset = offset || else_offset
# Jump to the offset of case or else
target_pc = jit.pc + (jit.insn.len + jump_offset) * C.VALUE.size
jit_direct_jump(jit.iseq, target_pc, ctx, asm)
EndBlock
else
KeepCompiling # continue with === branches
end
2023-03-03 21:10:17 -08:00
end
2023-02-06 15:44:34 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-06 15:44:34 -08:00
def opt_plus(jit, ctx, asm)
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
comptime_recv = jit.peek_at_stack(1)
comptime_obj = jit.peek_at_stack(0)
if fixnum?(comptime_recv) && fixnum?(comptime_obj)
# Generate a side exit before popping operands
side_exit = side_exit(jit, ctx)
unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_PLUS)
2023-02-06 15:44:34 -08:00
return CantCompile
end
obj_opnd = ctx.stack_pop
recv_opnd = ctx.stack_pop
asm.comment('guard recv is fixnum') # TODO: skip this with type information
asm.test(recv_opnd, C::RUBY_FIXNUM_FLAG)
2023-02-06 15:44:34 -08:00
asm.jz(side_exit)
asm.comment('guard obj is fixnum') # TODO: skip this with type information
asm.test(obj_opnd, C::RUBY_FIXNUM_FLAG)
2023-02-06 15:44:34 -08:00
asm.jz(side_exit)
asm.mov(:rax, recv_opnd)
asm.sub(:rax, 1) # untag
asm.mov(:rcx, obj_opnd)
asm.add(:rax, :rcx)
asm.jo(side_exit)
dst_opnd = ctx.stack_push
asm.mov(dst_opnd, :rax)
KeepCompiling
else
opt_send_without_block(jit, ctx, asm)
2023-02-06 15:44:34 -08:00
end
end
2023-01-07 14:06:38 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-01-07 14:06:38 -08:00
def opt_minus(jit, ctx, asm)
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
comptime_recv = jit.peek_at_stack(1)
comptime_obj = jit.peek_at_stack(0)
if fixnum?(comptime_recv) && fixnum?(comptime_obj)
2023-02-08 09:30:47 -08:00
# Generate a side exit before popping operands
side_exit = side_exit(jit, ctx)
unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_MINUS)
2023-01-07 14:06:38 -08:00
return CantCompile
end
2023-02-08 09:30:47 -08:00
obj_opnd = ctx.stack_pop
recv_opnd = ctx.stack_pop
2023-01-07 14:06:38 -08:00
asm.comment('guard recv is fixnum') # TODO: skip this with type information
asm.test(recv_opnd, C::RUBY_FIXNUM_FLAG)
2023-02-08 09:30:47 -08:00
asm.jz(side_exit)
2023-01-07 14:06:38 -08:00
asm.comment('guard obj is fixnum') # TODO: skip this with type information
asm.test(obj_opnd, C::RUBY_FIXNUM_FLAG)
2023-02-08 09:30:47 -08:00
asm.jz(side_exit)
2023-01-07 14:06:38 -08:00
2023-02-08 09:30:47 -08:00
asm.mov(:rax, recv_opnd)
asm.mov(:rcx, obj_opnd)
2023-01-07 14:06:38 -08:00
asm.sub(:rax, :rcx)
2023-02-08 09:30:47 -08:00
asm.jo(side_exit)
2023-02-06 15:44:34 -08:00
asm.add(:rax, 1) # re-tag
2023-01-07 14:06:38 -08:00
2023-02-08 09:30:47 -08:00
dst_opnd = ctx.stack_push
asm.mov(dst_opnd, :rax)
2023-01-07 14:06:38 -08:00
KeepCompiling
else
opt_send_without_block(jit, ctx, asm)
2023-01-07 14:06:38 -08:00
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_mult(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_div(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 22:36:02 -08:00
def opt_mod(jit, ctx, asm)
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
if two_fixnums_on_stack?(jit)
# Create a side-exit to fall back to the interpreter
# Note: we generate the side-exit before popping operands from the stack
side_exit = side_exit(jit, ctx)
unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_MOD)
2023-02-13 22:36:02 -08:00
return CantCompile
end
# Check that both operands are fixnums
guard_two_fixnums(jit, ctx, asm, side_exit)
# Get the operands and destination from the stack
arg1 = ctx.stack_pop(1)
arg0 = ctx.stack_pop(1)
# Check for arg0 % 0
asm.cmp(arg1, 0)
asm.je(side_exit)
# Call rb_fix_mod_fix(VALUE recv, VALUE obj)
asm.mov(C_ARGS[0], arg0)
asm.mov(C_ARGS[1], arg1)
asm.call(C.rb_fix_mod_fix)
# Push the return value onto the stack
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
else
opt_send_without_block(jit, ctx, asm)
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 23:57:40 -08:00
def opt_eq(jit, ctx, asm)
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
2023-03-02 23:00:52 -08:00
if jit_equality_specialized(jit, ctx, asm, true)
2023-02-13 23:57:40 -08:00
jump_to_next_insn(jit, ctx, asm)
EndBlock
else
opt_send_without_block(jit, ctx, asm)
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 23:57:40 -08:00
def opt_neq(jit, ctx, asm)
# opt_neq is passed two rb_call_data as arguments:
# first for ==, second for !=
neq_cd = C.rb_call_data.new(jit.operand(1))
2023-02-24 14:48:02 -08:00
opt_send_without_block(jit, ctx, asm, cd: neq_cd)
2023-02-13 23:57:40 -08:00
end
2022-12-31 13:41:32 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2022-12-31 13:41:32 -08:00
def opt_lt(jit, ctx, asm)
jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovl, bop: C::BOP_LT)
2023-02-13 21:48:24 -08:00
end
2023-01-02 22:53:14 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 21:48:24 -08:00
def opt_le(jit, ctx, asm)
jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovle, bop: C::BOP_LE)
2023-02-13 21:48:24 -08:00
end
2023-02-08 09:30:47 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 21:48:24 -08:00
def opt_gt(jit, ctx, asm)
jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovg, bop: C::BOP_GT)
2022-12-31 13:41:32 -08:00
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 21:48:24 -08:00
def opt_ge(jit, ctx, asm)
jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovge, bop: C::BOP_GE)
2023-02-13 21:48:24 -08:00
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_ltlt(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 23:05:56 -08:00
def opt_and(jit, ctx, asm)
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
if two_fixnums_on_stack?(jit)
# Create a side-exit to fall back to the interpreter
# Note: we generate the side-exit before popping operands from the stack
side_exit = side_exit(jit, ctx)
unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_AND)
2023-02-13 23:05:56 -08:00
return CantCompile
end
# Check that both operands are fixnums
guard_two_fixnums(jit, ctx, asm, side_exit)
# Get the operands and destination from the stack
arg1 = ctx.stack_pop(1)
arg0 = ctx.stack_pop(1)
asm.comment('bitwise and')
asm.mov(:rax, arg0)
asm.and(:rax, arg1)
# Push the return value onto the stack
dst = ctx.stack_push
asm.mov(dst, :rax)
KeepCompiling
else
opt_send_without_block(jit, ctx, asm)
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 23:05:56 -08:00
def opt_or(jit, ctx, asm)
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
if two_fixnums_on_stack?(jit)
# Create a side-exit to fall back to the interpreter
# Note: we generate the side-exit before popping operands from the stack
side_exit = side_exit(jit, ctx)
unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_OR)
2023-02-13 23:05:56 -08:00
return CantCompile
end
# Check that both operands are fixnums
guard_two_fixnums(jit, ctx, asm, side_exit)
# Get the operands and destination from the stack
asm.comment('bitwise or')
arg1 = ctx.stack_pop(1)
arg0 = ctx.stack_pop(1)
# Do the bitwise or arg0 | arg1
asm.mov(:rax, arg0)
asm.or(:rax, arg1)
# Push the return value onto the stack
dst = ctx.stack_push
asm.mov(dst, :rax)
KeepCompiling
else
opt_send_without_block(jit, ctx, asm)
end
end
2023-02-08 11:10:04 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 11:10:04 -08:00
def opt_aref(jit, ctx, asm)
cd = C.rb_call_data.new(jit.operand(0))
argc = C.vm_ci_argc(cd.ci)
if argc != 1
asm.incr_counter(:optaref_argc_not_one)
return CantCompile
end
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
comptime_recv = jit.peek_at_stack(1)
comptime_obj = jit.peek_at_stack(0)
side_exit = side_exit(jit, ctx)
2023-03-03 22:30:18 -08:00
if C.rb_class_of(comptime_recv) == Array && fixnum?(comptime_obj)
unless Invariants.assume_bop_not_redefined(jit, C::ARRAY_REDEFINED_OP_FLAG, C::BOP_AREF)
2023-02-13 22:23:22 -08:00
return CantCompile
end
idx_opnd = ctx.stack_opnd(0)
recv_opnd = ctx.stack_opnd(1)
not_array_exit = counted_exit(side_exit, :optaref_recv_not_array)
2023-03-03 22:03:23 -08:00
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv_opnd, comptime_recv, not_array_exit)
2023-02-13 22:23:22 -08:00
# Bail if idx is not a FIXNUM
asm.mov(:rax, idx_opnd)
asm.test(:rax, C::RUBY_FIXNUM_FLAG)
2023-02-13 22:23:22 -08:00
asm.jz(counted_exit(side_exit, :optaref_arg_not_fixnum))
# Call VALUE rb_ary_entry_internal(VALUE ary, long offset).
# It never raises or allocates, so we don't need to write to cfp->pc.
asm.sar(:rax, 1) # Convert fixnum to int
asm.mov(C_ARGS[0], recv_opnd)
asm.mov(C_ARGS[1], :rax)
asm.call(C.rb_ary_entry_internal)
# Pop the argument and the receiver
ctx.stack_pop(2)
# Push the return value onto the stack
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
# Let guard chains share the same successor
jump_to_next_insn(jit, ctx, asm)
EndBlock
2023-03-03 22:30:18 -08:00
elsif C.rb_class_of(comptime_recv) == Hash
unless Invariants.assume_bop_not_redefined(jit, C::HASH_REDEFINED_OP_FLAG, C::BOP_AREF)
2023-02-08 11:10:04 -08:00
return CantCompile
end
recv_opnd = ctx.stack_opnd(1)
# Guard that the receiver is a Hash
2023-02-13 22:23:22 -08:00
not_hash_exit = counted_exit(side_exit, :optaref_recv_not_hash)
2023-03-03 22:03:23 -08:00
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv_opnd, comptime_recv, not_hash_exit)
2023-02-08 11:10:04 -08:00
# Prepare to call rb_hash_aref(). It might call #hash on the key.
jit_prepare_routine_call(jit, ctx, asm)
asm.comment('call rb_hash_aref')
key_opnd = ctx.stack_opnd(0)
recv_opnd = ctx.stack_opnd(1)
asm.mov(:rdi, recv_opnd)
asm.mov(:rsi, key_opnd)
asm.call(C.rb_hash_aref)
# Pop the key and the receiver
ctx.stack_pop(2)
stack_ret = ctx.stack_push
2023-02-10 22:07:34 -08:00
asm.mov(stack_ret, C_RET)
2023-02-08 11:10:04 -08:00
# Let guard chains share the same successor
jump_to_next_insn(jit, ctx, asm)
EndBlock
else
opt_send_without_block(jit, ctx, asm)
2023-02-08 11:10:04 -08:00
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-15 23:13:14 -08:00
def opt_aset(jit, ctx, asm)
# Defer compilation so we can specialize on a runtime `self`
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
comptime_recv = jit.peek_at_stack(2)
comptime_key = jit.peek_at_stack(1)
# Get the operands from the stack
recv = ctx.stack_opnd(2)
key = ctx.stack_opnd(1)
_val = ctx.stack_opnd(0)
2023-03-03 22:30:18 -08:00
if C.rb_class_of(comptime_recv) == Array && fixnum?(comptime_key)
2023-02-15 23:13:14 -08:00
side_exit = side_exit(jit, ctx)
# Guard receiver is an Array
2023-03-03 22:03:23 -08:00
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, comptime_recv, side_exit)
2023-02-15 23:13:14 -08:00
# Guard key is a fixnum
2023-03-03 22:03:23 -08:00
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_key), key, comptime_key, side_exit)
2023-02-15 23:13:14 -08:00
# We might allocate or raise
jit_prepare_routine_call(jit, ctx, asm)
asm.comment('call rb_ary_store')
recv = ctx.stack_opnd(2)
key = ctx.stack_opnd(1)
val = ctx.stack_opnd(0)
asm.mov(:rax, key)
asm.sar(:rax, 1) # FIX2LONG(key)
asm.mov(C_ARGS[0], recv)
asm.mov(C_ARGS[1], :rax)
asm.mov(C_ARGS[2], val)
asm.call(C.rb_ary_store)
# rb_ary_store returns void
# stored value should still be on stack
val = ctx.stack_opnd(0)
# Push the return value onto the stack
ctx.stack_pop(3)
stack_ret = ctx.stack_push
asm.mov(:rax, val)
asm.mov(stack_ret, :rax)
jump_to_next_insn(jit, ctx, asm)
EndBlock
2023-03-03 22:30:18 -08:00
elsif C.rb_class_of(comptime_recv) == Hash
2023-02-15 23:13:14 -08:00
side_exit = side_exit(jit, ctx)
# Guard receiver is a Hash
2023-03-03 22:03:23 -08:00
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, comptime_recv, side_exit)
2023-02-15 23:13:14 -08:00
# We might allocate or raise
jit_prepare_routine_call(jit, ctx, asm)
# Call rb_hash_aset
recv = ctx.stack_opnd(2)
key = ctx.stack_opnd(1)
val = ctx.stack_opnd(0)
asm.mov(C_ARGS[0], recv)
asm.mov(C_ARGS[1], key)
asm.mov(C_ARGS[2], val)
asm.call(C.rb_hash_aset)
# Push the return value onto the stack
ctx.stack_pop(3)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
jump_to_next_insn(jit, ctx, asm)
EndBlock
else
opt_send_without_block(jit, ctx, asm)
end
end
2022-12-28 13:50:24 -08:00
# opt_aset_with
# opt_aref_with
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_length(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_size(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_empty_p(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_succ(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_not(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def opt_regexpmatch2(jit, ctx, asm)
opt_send_without_block(jit, ctx, asm)
end
2022-12-28 13:50:24 -08:00
# invokebuiltin
2023-02-16 00:11:38 -08:00
def opt_invokebuiltin_delegate(jit, ctx, asm)
bf = C.rb_builtin_function.new(jit.operand(0))
bf_argc = bf.argc
start_index = jit.operand(1)
# ec, self, and arguments
if bf_argc + 2 > C_ARGS.size
return CantCompile
end
# If the calls don't allocate, do they need up to date PC, SP?
jit_prepare_routine_call(jit, ctx, asm)
# Call the builtin func (ec, recv, arg1, arg2, ...)
asm.comment('call builtin func')
asm.mov(C_ARGS[0], EC)
asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:self)])
# Copy arguments from locals
if bf_argc > 0
# Load environment pointer EP from CFP
asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:ep)])
bf_argc.times do |i|
table_size = jit.iseq.body.local_table_size
offs = -table_size - C::VM_ENV_DATA_SIZE + 1 + start_index + i
2023-02-16 00:11:38 -08:00
asm.mov(C_ARGS[2 + i], [:rax, offs * C.VALUE.size])
end
end
asm.call(bf.func_ptr)
# Push the return value
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-16 00:11:38 -08:00
def opt_invokebuiltin_delegate_leave(jit, ctx, asm)
opt_invokebuiltin_delegate(jit, ctx, asm)
# opt_invokebuiltin_delegate is always followed by leave insn
end
2022-12-28 13:50:24 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2022-12-28 13:50:24 -08:00
def getlocal_WC_0(jit, ctx, asm)
# Get operands
idx = jit.operand(0)
# Get EP
asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:ep)])
# Get a local variable
asm.mov(:rax, [:rax, -idx * C.VALUE.size])
# Push it to the stack
2023-02-08 01:17:39 -08:00
stack_top = ctx.stack_push
asm.mov(stack_top, :rax)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 01:17:39 -08:00
def getlocal_WC_1(jit, ctx, asm)
idx = jit.operand(0)
2023-02-14 22:24:32 -08:00
jit_getlocal_generic(jit, ctx, asm, idx:, level: 1)
2022-12-28 13:50:24 -08:00
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 22:50:52 -08:00
def setlocal_WC_0(jit, ctx, asm)
slot_idx = jit.operand(0)
# Load environment pointer EP (level 0) from CFP
ep_reg = :rax
jit_get_ep(asm, 0, reg: ep_reg)
# Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers
# only affect heap objects being written. If we know an immediate value is being written we
# can skip this check.
# flags & VM_ENV_FLAG_WB_REQUIRED
flags_opnd = [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS]
asm.test(flags_opnd, C::VM_ENV_FLAG_WB_REQUIRED)
2023-02-13 22:50:52 -08:00
# Create a side-exit to fall back to the interpreter
side_exit = side_exit(jit, ctx)
# if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0
asm.jnz(side_exit)
# Pop the value to write from the stack
stack_top = ctx.stack_pop(1)
# Write the value at the environment pointer
asm.mov(:rcx, stack_top)
asm.mov([ep_reg, -8 * slot_idx], :rcx)
KeepCompiling
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-14 22:24:32 -08:00
def setlocal_WC_1(jit, ctx, asm)
idx = jit.operand(0)
jit_setlocal_generic(jit, ctx, asm, idx:, level: 1)
end
2023-01-07 13:54:26 -08:00
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-01-07 13:54:26 -08:00
def putobject_INT2FIX_0_(jit, ctx, asm)
putobject(jit, ctx, asm, val: C.to_value(0))
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-01-07 13:54:26 -08:00
def putobject_INT2FIX_1_(jit, ctx, asm)
putobject(jit, ctx, asm, val: C.to_value(1))
end
2022-12-28 13:50:24 -08:00
2023-03-02 22:23:47 -08:00
#
# C func
#
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:55:38 -08:00
def jit_rb_true(jit, ctx, asm, argc, _known_recv_class)
return false if argc != 0
asm.comment('nil? == true');
ctx.stack_pop(1)
stack_ret = ctx.stack_push
asm.mov(stack_ret, Qtrue)
true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:55:38 -08:00
def jit_rb_false(jit, ctx, asm, argc, _known_recv_class)
return false if argc != 0
asm.comment('nil? == false');
ctx.stack_pop(1)
stack_ret = ctx.stack_push
asm.mov(stack_ret, Qfalse)
true
end
2023-03-18 22:57:31 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_rb_kernel_is_a(jit, ctx, asm, argc, known_recv_class)
if argc != 1
return false
end
# If this is a super call we might not know the class
if known_recv_class.nil?
return false
end
# Important note: The output code will simply `return true/false`.
# Correctness follows from:
# - `known_recv_class` implies there is a guard scheduled before here
# for a particular `CLASS_OF(lhs)`.
# - We guard that rhs is identical to the compile-time sample
# - In general, for any two Class instances A, B, `A < B` does not change at runtime.
# Class#superclass is stable.
sample_rhs = jit.peek_at_stack(0)
sample_lhs = jit.peek_at_stack(1)
# We are not allowing module here because the module hierachy can change at runtime.
if C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS)
return false
end
sample_is_a = C.obj_is_kind_of(sample_lhs, sample_rhs)
side_exit = side_exit(jit, ctx)
asm.comment('Kernel#is_a?')
asm.mov(:rax, to_value(sample_rhs))
asm.cmp(ctx.stack_opnd(0), :rax)
asm.jne(counted_exit(side_exit, :send_is_a_class_mismatch))
ctx.stack_pop(2)
stack_ret = ctx.stack_push
if sample_is_a
asm.mov(stack_ret, Qtrue)
else
asm.mov(stack_ret, Qfalse)
end
return true
end
2023-03-18 23:13:25 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_rb_kernel_instance_of(jit, ctx, asm, argc, known_recv_class)
if argc != 1
return false
end
# If this is a super call we might not know the class
if known_recv_class.nil?
return false
end
# Important note: The output code will simply `return true/false`.
# Correctness follows from:
# - `known_recv_class` implies there is a guard scheduled before here
# for a particular `CLASS_OF(lhs)`.
# - We guard that rhs is identical to the compile-time sample
# - For a particular `CLASS_OF(lhs)`, `rb_obj_class(lhs)` does not change.
# (because for any singleton class `s`, `s.superclass.equal?(s.attached_object.class)`)
sample_rhs = jit.peek_at_stack(0)
sample_lhs = jit.peek_at_stack(1)
# Filters out cases where the C implementation raises
unless C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS) || C.RB_TYPE_P(sample_rhs, C::RUBY_T_MODULE)
return false
end
# We need to grab the class here to deal with singleton classes.
# Instance of grabs the "real class" of the object rather than the
# singleton class.
sample_lhs_real_class = C.rb_obj_class(sample_lhs)
sample_instance_of = (sample_lhs_real_class == sample_rhs)
side_exit = side_exit(jit, ctx)
asm.comment('Kernel#instance_of?')
asm.mov(:rax, to_value(sample_rhs))
asm.cmp(ctx.stack_opnd(0), :rax)
asm.jne(counted_exit(side_exit, :send_instance_of_class_mismatch))
ctx.stack_pop(2)
stack_ret = ctx.stack_push
if sample_instance_of
asm.mov(stack_ret, Qtrue)
else
asm.mov(stack_ret, Qfalse)
end
return true;
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_rb_obj_not(jit, ctx, asm, argc, _known_recv_class)
2023-03-02 22:43:55 -08:00
return false if argc != 0
2023-03-02 22:52:32 -08:00
asm.comment('rb_obj_not')
2023-03-02 22:43:55 -08:00
2023-03-02 22:23:47 -08:00
recv = ctx.stack_pop
# This `test` sets ZF only for Qnil and Qfalse, which let cmovz set.
asm.test(recv, ~Qnil)
asm.mov(:rax, Qfalse)
asm.mov(:rcx, Qtrue)
asm.cmovz(:rax, :rcx)
2023-03-02 23:21:28 -08:00
stack_ret = ctx.stack_push
asm.mov(stack_ret, :rax)
2023-03-02 22:23:47 -08:00
true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_rb_obj_equal(jit, ctx, asm, argc, _known_recv_class)
2023-03-03 23:36:34 -08:00
return false if argc != 1
asm.comment('equal?')
obj1 = ctx.stack_pop(1)
obj2 = ctx.stack_pop(1)
asm.mov(:rax, obj1)
asm.mov(:rcx, obj2)
asm.cmp(:rax, :rcx)
asm.mov(:rax, Qfalse)
asm.mov(:rcx, Qtrue)
asm.cmove(:rax, :rcx)
stack_ret = ctx.stack_push
asm.mov(stack_ret, :rax)
true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_rb_obj_not_equal(jit, ctx, asm, argc, _known_recv_class)
2023-03-02 23:00:52 -08:00
return false if argc != 1
jit_equality_specialized(jit, ctx, asm, false)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-04 23:44:11 -08:00
def jit_rb_mod_eqq(jit, ctx, asm, argc, _known_recv_class)
return false if argc != 1
asm.comment('Module#===')
# By being here, we know that the receiver is a T_MODULE or a T_CLASS, because Module#=== can
# only live on these objects. With that, we can call rb_obj_is_kind_of() without
# jit_prepare_routine_call() or a control frame push because it can't raise, allocate, or call
# Ruby methods with these inputs.
# Note the difference in approach from Kernel#is_a? because we don't get a free guard for the
# right hand side.
lhs = ctx.stack_opnd(1) # the module
rhs = ctx.stack_opnd(0)
asm.mov(C_ARGS[0], rhs);
asm.mov(C_ARGS[1], lhs);
asm.call(C.rb_obj_is_kind_of)
# Return the result
ctx.stack_pop(2)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
return true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_rb_int_equal(jit, ctx, asm, argc, _known_recv_class)
2023-03-02 22:52:32 -08:00
return false if argc != 1
return false unless two_fixnums_on_stack?(jit)
side_exit = side_exit(jit, ctx)
guard_two_fixnums(jit, ctx, asm, side_exit)
# Compare the arguments
asm.comment('rb_int_equal')
arg1 = ctx.stack_pop(1)
arg0 = ctx.stack_pop(1)
asm.mov(:rax, arg1)
asm.cmp(arg0, :rax)
asm.mov(:rax, Qfalse)
asm.mov(:rcx, Qtrue)
asm.cmove(:rax, :rcx)
stack_ret = ctx.stack_push
asm.mov(stack_ret, :rax)
true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_rb_int_mul(jit, ctx, asm, argc, _known_recv_class)
2023-03-02 22:43:55 -08:00
return false if argc != 1
return false unless two_fixnums_on_stack?(jit)
side_exit = side_exit(jit, ctx)
guard_two_fixnums(jit, ctx, asm, side_exit)
2023-03-02 23:21:28 -08:00
asm.comment('rb_int_mul')
2023-03-02 22:43:55 -08:00
y_opnd = ctx.stack_pop
x_opnd = ctx.stack_pop
asm.mov(C_ARGS[0], x_opnd)
asm.mov(C_ARGS[1], y_opnd)
asm.call(C.rb_fix_mul_fix)
ret_opnd = ctx.stack_push
asm.mov(ret_opnd, C_RET)
2023-03-02 23:21:28 -08:00
true
end
2023-03-03 23:51:31 -08:00
def jit_rb_int_div(jit, ctx, asm, argc, _known_recv_class)
2023-03-02 23:21:28 -08:00
return false if argc != 1
return false unless two_fixnums_on_stack?(jit)
2023-03-02 22:43:55 -08:00
2023-03-02 23:21:28 -08:00
side_exit = side_exit(jit, ctx)
guard_two_fixnums(jit, ctx, asm, side_exit)
asm.comment('rb_int_div')
y_opnd = ctx.stack_pop
x_opnd = ctx.stack_pop
asm.mov(:rax, y_opnd)
asm.cmp(:rax, C.to_value(0))
asm.je(side_exit)
asm.mov(C_ARGS[0], x_opnd)
asm.mov(C_ARGS[1], :rax)
asm.call(C.rb_fix_div_fix)
ret_opnd = ctx.stack_push
asm.mov(ret_opnd, C_RET)
2023-03-02 22:43:55 -08:00
true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_rb_int_aref(jit, ctx, asm, argc, _known_recv_class)
2023-03-02 23:28:24 -08:00
return false if argc != 1
return false unless two_fixnums_on_stack?(jit)
side_exit = side_exit(jit, ctx)
guard_two_fixnums(jit, ctx, asm, side_exit)
asm.comment('rb_int_aref')
y_opnd = ctx.stack_pop
x_opnd = ctx.stack_pop
asm.mov(C_ARGS[0], x_opnd)
asm.mov(C_ARGS[1], y_opnd)
asm.call(C.rb_fix_aref)
ret_opnd = ctx.stack_push
asm.mov(ret_opnd, C_RET)
true
end
2023-03-18 23:24:57 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_rb_str_empty_p(jit, ctx, asm, argc, known_recv_class)
# Assume same offset to len embedded or not so we can use one code path to read the length
2023-03-18 23:42:58 -07:00
#assert_equal(C.RString.offsetof(:as, :heap, :len), C.RString.offsetof(:as, :embed, :len))
# `C.RString.offsetof(:as, :embed, :len)` doesn't work because of USE_RVARGC=0 CI
2023-03-18 23:24:57 -07:00
recv_opnd = ctx.stack_pop(1)
out_opnd = ctx.stack_push
asm.comment('get string length')
asm.mov(:rax, recv_opnd)
str_len_opnd = [:rax, C.RString.offsetof(:as, :heap, :len)]
asm.cmp(str_len_opnd, 0)
asm.mov(:rax, Qfalse)
asm.mov(:rcx, Qtrue)
asm.cmove(:rax, :rcx)
asm.mov(out_opnd, :rax)
return true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_rb_str_to_s(jit, ctx, asm, argc, known_recv_class)
return false if argc != 0
if known_recv_class == String
asm.comment('to_s on plain string')
# The method returns the receiver, which is already on the stack.
# No stack movement.
return true
end
false
end
2023-03-18 23:33:10 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_rb_str_bytesize(jit, ctx, asm, argc, known_recv_class)
asm.comment('String#bytesize')
recv = ctx.stack_pop(1)
asm.mov(C_ARGS[0], recv)
asm.call(C.rb_str_bytesize)
out_opnd = ctx.stack_push
asm.mov(out_opnd, C_RET)
true
end
2023-03-18 23:49:11 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_rb_str_concat(jit, ctx, asm, argc, known_recv_class)
# The << operator can accept integer codepoints for characters
# as the argument. We only specially optimise string arguments.
# If the peeked-at compile time argument is something other than
# a string, assume it won't be a string later either.
comptime_arg = jit.peek_at_stack(0)
unless C.RB_TYPE_P(comptime_arg, C::RUBY_T_STRING)
return false
end
# Generate a side exit
side_exit = side_exit(jit, ctx)
# Guard that the concat argument is a string
asm.mov(:rax, ctx.stack_opnd(0))
guard_object_is_string(asm, :rax, :rcx, side_exit)
# Guard buffers from GC since rb_str_buf_append may allocate.
jit_save_sp(ctx, asm)
concat_arg = ctx.stack_pop(1)
recv = ctx.stack_pop(1)
# Test if string encodings differ. If different, use rb_str_append. If the same,
# use rb_yjit_str_simple_append, which calls rb_str_cat.
asm.comment('<< on strings')
# Take receiver's object flags XOR arg's flags. If any
# string-encoding flags are different between the two,
# the encodings don't match.
recv_reg = :rax
asm.mov(recv_reg, recv)
concat_arg_reg = :rcx
asm.mov(concat_arg_reg, concat_arg)
asm.mov(recv_reg, [recv_reg, C.RBasic.offsetof(:flags)])
asm.mov(concat_arg_reg, [concat_arg_reg, C.RBasic.offsetof(:flags)])
asm.xor(recv_reg, concat_arg_reg)
asm.test(recv_reg, C::RUBY_ENCODING_MASK)
# Push once, use the resulting operand in both branches below.
stack_ret = ctx.stack_push
enc_mismatch = asm.new_label('enc_mismatch')
asm.jnz(enc_mismatch)
# If encodings match, call the simple append function and jump to return
asm.mov(C_ARGS[0], recv)
asm.mov(C_ARGS[1], concat_arg)
asm.call(C.rjit_str_simple_append)
ret_label = asm.new_label('func_return')
asm.mov(stack_ret, C_RET)
asm.jmp(ret_label)
# If encodings are different, use a slower encoding-aware concatenate
asm.write_label(enc_mismatch)
asm.mov(C_ARGS[0], recv)
asm.mov(C_ARGS[1], concat_arg)
asm.call(C.rb_str_buf_append)
asm.mov(stack_ret, C_RET)
# Drop through to return
asm.write_label(ret_label)
true
end
2023-03-19 13:36:26 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_rb_str_uplus(jit, ctx, asm, argc, _known_recv_class)
if argc != 0
return false
end
# We allocate when we dup the string
jit_prepare_routine_call(jit, ctx, asm)
asm.comment('Unary plus on string')
asm.mov(:rax, ctx.stack_pop(1)) # recv_opnd
asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)]) # flags_opnd
asm.test(:rcx, C::RUBY_FL_FREEZE)
ret_label = asm.new_label('stack_ret')
# String#+@ can only exist on T_STRING
stack_ret = ctx.stack_push
# If the string isn't frozen, we just return it.
asm.mov(stack_ret, :rax) # recv_opnd
asm.jz(ret_label)
# Str is frozen - duplicate it
asm.mov(C_ARGS[0], :rax) # recv_opnd
asm.call(C.rb_str_dup)
asm.mov(stack_ret, C_RET)
asm.write_label(ret_label)
true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-04 23:53:21 -08:00
def jit_rb_str_getbyte(jit, ctx, asm, argc, _known_recv_class)
return false if argc != 1
asm.comment('rb_str_getbyte')
index_opnd = ctx.stack_pop
str_opnd = ctx.stack_pop
asm.mov(C_ARGS[0], str_opnd)
asm.mov(C_ARGS[1], index_opnd)
asm.call(C.rb_str_getbyte)
ret_opnd = ctx.stack_push
asm.mov(ret_opnd, C_RET)
true
end
2023-03-19 13:41:04 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_rb_ary_empty_p(jit, ctx, asm, argc, _known_recv_class)
array_reg = :rax
asm.mov(array_reg, ctx.stack_pop(1))
2023-03-19 20:27:55 -07:00
jit_array_len(asm, array_reg, :rcx)
2023-03-19 13:41:04 -07:00
asm.test(:rcx, :rcx)
asm.mov(:rax, Qfalse)
asm.mov(:rcx, Qtrue)
asm.cmovz(:rax, :rcx)
out_opnd = ctx.stack_push
asm.mov(out_opnd, :rax)
return true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_rb_ary_push(jit, ctx, asm, argc, _known_recv_class)
2023-03-02 23:15:02 -08:00
return false if argc != 1
asm.comment('rb_ary_push')
jit_prepare_routine_call(jit, ctx, asm)
item_opnd = ctx.stack_pop
ary_opnd = ctx.stack_pop
asm.mov(C_ARGS[0], ary_opnd)
asm.mov(C_ARGS[1], item_opnd)
asm.call(C.rb_ary_push)
ret_opnd = ctx.stack_push
asm.mov(ret_opnd, C_RET)
true
end
2023-03-19 13:46:09 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_obj_respond_to(jit, ctx, asm, argc, known_recv_class)
# respond_to(:sym) or respond_to(:sym, true)
if argc != 1 && argc != 2
return false
end
if known_recv_class.nil?
return false
end
recv_class = known_recv_class
# Get the method_id from compile time. We will later add a guard against it.
mid_sym = jit.peek_at_stack(argc - 1)
unless static_symbol?(mid_sym)
return false
end
mid = C.rb_sym2id(mid_sym)
target_cme = C.rb_callable_method_entry_or_negative(recv_class, mid)
# Should never be null, as in that case we will be returned a "negative CME"
assert_equal(false, target_cme.nil?)
cme_def_type =
if C.UNDEFINED_METHOD_ENTRY_P(target_cme)
C::VM_METHOD_TYPE_UNDEF
else
target_cme.def.type
end
if cme_def_type == C::VM_METHOD_TYPE_REFINED
return false
end
visibility = if cme_def_type == C::VM_METHOD_TYPE_UNDEF
C::METHOD_VISI_UNDEF
else
C.METHOD_ENTRY_VISI(target_cme)
end
result =
case visibility
in C::METHOD_VISI_UNDEF
Qfalse # No method => false
in C::METHOD_VISI_PUBLIC
Qtrue # Public method => true regardless of include_all
else
return false # not public and include_all not known, can't compile
end
if result != Qtrue
# Only if respond_to_missing? hasn't been overridden
# In the future, we might want to jit the call to respond_to_missing?
unless Invariants.assume_method_basic_definition(jit, recv_class, C.idRespond_to_missing)
return false
end
end
# Invalidate this block if method lookup changes for the method being queried. This works
# both for the case where a method does or does not exist, as for the latter we asked for a
# "negative CME" earlier.
Invariants.assume_method_lookup_stable(jit, target_cme)
# Generate a side exit
side_exit = side_exit(jit, ctx)
if argc == 2
# pop include_all argument (we only use its type info)
ctx.stack_pop(1)
end
sym_opnd = ctx.stack_pop(1)
_recv_opnd = ctx.stack_pop(1)
# This is necessary because we have no guarantee that sym_opnd is a constant
asm.comment('guard known mid')
asm.mov(:rax, to_value(mid_sym))
asm.cmp(sym_opnd, :rax)
asm.jne(side_exit)
putobject(jit, ctx, asm, val: result)
true
end
2023-03-19 14:15:45 -07:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_rb_f_block_given_p(jit, ctx, asm, argc, _known_recv_class)
asm.comment('block_given?')
# Same as rb_vm_frame_block_handler
jit_get_lep(jit, asm, reg: :rax)
asm.mov(:rax, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler
ctx.stack_pop(1)
out_opnd = ctx.stack_push
# Return `block_handler != VM_BLOCK_HANDLER_NONE`
asm.cmp(:rax, C::VM_BLOCK_HANDLER_NONE)
asm.mov(:rax, Qfalse)
asm.mov(:rcx, Qtrue)
asm.cmovne(:rax, :rcx) # block_given
asm.mov(out_opnd, :rax)
true
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-04 00:01:30 -08:00
def jit_thread_s_current(jit, ctx, asm, argc, _known_recv_class)
return false if argc != 0
asm.comment('Thread.current')
ctx.stack_pop(1)
# ec->thread_ptr
asm.mov(:rax, [EC, C.rb_execution_context_t.offsetof(:thread_ptr)])
# thread->self
asm.mov(:rax, [:rax, C.rb_thread_struct.offsetof(:self)])
stack_ret = ctx.stack_push
asm.mov(stack_ret, :rax)
true
end
#
# Helpers
#
2022-12-23 14:17:32 -08:00
2023-03-02 22:23:47 -08:00
def register_cfunc_codegen_funcs
2023-03-03 23:36:34 -08:00
# Specialization for C methods. See register_cfunc_method for details.
2023-03-02 22:43:55 -08:00
register_cfunc_method(BasicObject, :!, :jit_rb_obj_not)
2023-03-02 22:52:32 -08:00
2023-03-03 23:55:38 -08:00
register_cfunc_method(NilClass, :nil?, :jit_rb_true)
register_cfunc_method(Kernel, :nil?, :jit_rb_false)
2023-03-18 22:57:31 -07:00
register_cfunc_method(Kernel, :is_a?, :jit_rb_kernel_is_a)
register_cfunc_method(Kernel, :kind_of?, :jit_rb_kernel_is_a)
2023-03-18 23:13:25 -07:00
register_cfunc_method(Kernel, :instance_of?, :jit_rb_kernel_instance_of)
2023-03-03 23:36:34 -08:00
register_cfunc_method(BasicObject, :==, :jit_rb_obj_equal)
register_cfunc_method(BasicObject, :equal?, :jit_rb_obj_equal)
2023-03-02 23:00:52 -08:00
register_cfunc_method(BasicObject, :!=, :jit_rb_obj_not_equal)
2023-03-03 23:36:34 -08:00
register_cfunc_method(Kernel, :eql?, :jit_rb_obj_equal)
register_cfunc_method(Module, :==, :jit_rb_obj_equal)
2023-03-04 23:44:11 -08:00
register_cfunc_method(Module, :===, :jit_rb_mod_eqq)
2023-03-03 23:36:34 -08:00
register_cfunc_method(Symbol, :==, :jit_rb_obj_equal)
register_cfunc_method(Symbol, :===, :jit_rb_obj_equal)
2023-03-02 22:52:32 -08:00
register_cfunc_method(Integer, :==, :jit_rb_int_equal)
register_cfunc_method(Integer, :===, :jit_rb_int_equal)
2023-03-03 23:36:34 -08:00
# rb_str_to_s() methods in string.c
2023-03-18 23:24:57 -07:00
register_cfunc_method(String, :empty?, :jit_rb_str_empty_p)
2023-03-03 23:51:31 -08:00
register_cfunc_method(String, :to_s, :jit_rb_str_to_s)
register_cfunc_method(String, :to_str, :jit_rb_str_to_s)
2023-03-18 23:33:10 -07:00
register_cfunc_method(String, :bytesize, :jit_rb_str_bytesize)
2023-03-18 23:49:11 -07:00
register_cfunc_method(String, :<<, :jit_rb_str_concat)
2023-03-19 13:36:26 -07:00
register_cfunc_method(String, :+@, :jit_rb_str_uplus)
2023-03-03 23:36:34 -08:00
# rb_ary_empty_p() method in array.c
2023-03-19 13:41:04 -07:00
register_cfunc_method(Array, :empty?, :jit_rb_ary_empty_p)
2023-03-03 23:36:34 -08:00
2023-03-19 13:46:09 -07:00
register_cfunc_method(Kernel, :respond_to?, :jit_obj_respond_to)
2023-03-19 14:15:45 -07:00
register_cfunc_method(Kernel, :block_given?, :jit_rb_f_block_given_p)
2023-03-03 23:36:34 -08:00
# Thread.current
2023-03-04 00:01:30 -08:00
register_cfunc_method(C.rb_singleton_class(Thread), :current, :jit_thread_s_current)
2023-03-03 23:36:34 -08:00
#---
register_cfunc_method(Array, :<<, :jit_rb_ary_push)
2023-03-02 22:43:55 -08:00
register_cfunc_method(Integer, :*, :jit_rb_int_mul)
2023-03-02 23:21:28 -08:00
register_cfunc_method(Integer, :/, :jit_rb_int_div)
2023-03-02 23:28:24 -08:00
register_cfunc_method(Integer, :[], :jit_rb_int_aref)
2023-03-04 23:53:21 -08:00
register_cfunc_method(String, :getbyte, :jit_rb_str_getbyte)
2023-03-02 22:23:47 -08:00
end
2023-03-02 22:43:55 -08:00
def register_cfunc_method(klass, mid_sym, func)
mid = C.rb_intern(mid_sym.to_s)
2023-03-02 22:23:47 -08:00
me = C.rb_method_entry_at(klass, mid)
assert_equal(false, me.nil?)
# Only cfuncs are supported
method_serial = me.def.method_serial
@cfunc_codegen_table[method_serial] = method(func)
end
def lookup_cfunc_codegen(cme_def)
@cfunc_codegen_table[cme_def.method_serial]
end
2023-02-14 22:24:32 -08:00
def jit_getlocal_generic(jit, ctx, asm, idx:, level:)
# Load environment pointer EP at level
ep_reg = :rax
jit_get_ep(asm, level, reg: ep_reg)
# Get a local variable
asm.mov(:rax, [ep_reg, -idx * C.VALUE.size])
# Push it to the stack
stack_top = ctx.stack_push
asm.mov(stack_top, :rax)
KeepCompiling
end
def jit_setlocal_generic(jit, ctx, asm, idx:, level:)
# Load environment pointer EP at level
ep_reg = :rax
jit_get_ep(asm, level, reg: ep_reg)
# Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers
# only affect heap objects being written. If we know an immediate value is being written we
# can skip this check.
# flags & VM_ENV_FLAG_WB_REQUIRED
flags_opnd = [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS]
asm.test(flags_opnd, C::VM_ENV_FLAG_WB_REQUIRED)
2023-02-14 22:24:32 -08:00
# Create a side-exit to fall back to the interpreter
side_exit = side_exit(jit, ctx)
# if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0
asm.jnz(side_exit)
# Pop the value to write from the stack
stack_top = ctx.stack_pop(1)
# Write the value at the environment pointer
asm.mov(:rcx, stack_top)
asm.mov([ep_reg, -(C.VALUE.size * idx)], :rcx)
KeepCompiling
end
2023-02-13 22:50:52 -08:00
# Compute the index of a local variable from its slot index
def slot_to_local_idx(iseq, slot_idx)
# Layout illustration
# This is an array of VALUE
# | VM_ENV_DATA_SIZE |
# v v
# low addr <+-------+-------+-------+-------+------------------+
# |local 0|local 1| ... |local n| .... |
# +-------+-------+-------+-------+------------------+
# ^ ^ ^ ^
# +-------+---local_table_size----+ cfp->ep--+
# | |
# +------------------slot_idx----------------+
#
# See usages of local_var_name() from iseq.c for similar calculation.
local_table_size = iseq.body.local_table_size
op = slot_idx - C::VM_ENV_DATA_SIZE
2023-02-13 22:50:52 -08:00
local_table_size - op - 1
end
2023-03-06 23:15:30 -08:00
# @param asm [RubyVM::RJIT::Assembler]
2023-02-07 14:42:58 -08:00
def guard_object_is_heap(asm, object_opnd, side_exit)
asm.comment('guard object is heap')
# Test that the object is not an immediate
asm.test(object_opnd, C::RUBY_IMMEDIATE_MASK)
2023-02-07 14:42:58 -08:00
asm.jnz(side_exit)
# Test that the object is not false
asm.cmp(object_opnd, Qfalse)
asm.je(side_exit)
end
2023-03-06 23:15:30 -08:00
# @param asm [RubyVM::RJIT::Assembler]
2023-02-15 21:26:04 -08:00
def guard_object_is_array(asm, object_reg, flags_reg, side_exit)
asm.comment('guard object is array')
# Pull out the type mask
asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)])
asm.and(flags_reg, C::RUBY_T_MASK)
2023-02-15 21:26:04 -08:00
# Compare the result with T_ARRAY
asm.cmp(flags_reg, C::RUBY_T_ARRAY)
2023-02-15 21:26:04 -08:00
asm.jne(side_exit)
end
2023-03-18 23:49:11 -07:00
def guard_object_is_string(asm, object_reg, flags_reg, side_exit)
asm.comment('guard object is string')
# Pull out the type mask
asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)])
asm.and(flags_reg, C::RUBY_T_MASK)
# Compare the result with T_STRING
asm.cmp(flags_reg, C::RUBY_T_STRING)
asm.jne(side_exit)
end
2023-03-21 22:55:23 -07:00
# clobbers object_reg
def guard_object_is_not_ruby2_keyword_hash(asm, object_reg, flags_reg, side_exit)
asm.comment('guard object is not ruby2 keyword hash')
not_ruby2_keyword = asm.new_label('not_ruby2_keyword')
asm.test(object_reg, C::RUBY_IMMEDIATE_MASK)
asm.jnz(not_ruby2_keyword)
asm.cmp(object_reg, Qfalse)
asm.je(not_ruby2_keyword)
asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)])
type_reg = object_reg
asm.mov(type_reg, flags_reg)
asm.and(type_reg, C::RUBY_T_MASK)
asm.cmp(type_reg, C::RUBY_T_HASH)
asm.jne(not_ruby2_keyword)
asm.test(flags_reg, C::RHASH_PASS_AS_KEYWORDS)
asm.jnz(side_exit)
asm.write_label(not_ruby2_keyword)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-01 23:29:32 -08:00
def jit_chain_guard(opcode, jit, ctx, asm, side_exit, limit: 20)
2023-02-13 21:48:24 -08:00
opcode => :je | :jne | :jnz | :jz
2023-02-08 00:47:01 -08:00
if ctx.chain_depth < limit
deeper = ctx.dup
deeper.chain_depth += 1
branch_stub = BranchStub.new(
iseq: jit.iseq,
shape: Default,
target0: BranchTarget.new(ctx: deeper, pc: jit.pc),
)
branch_stub.target0.address = Assembler.new.then do |ocb_asm|
@exit_compiler.compile_branch_stub(deeper, ocb_asm, branch_stub, true)
@ocb.write(ocb_asm)
end
branch_stub.compile = proc do |branch_asm|
2023-02-09 16:25:06 -08:00
# Not using `asm.comment` here since it's usually put before cmp/test before this.
2023-02-08 00:47:01 -08:00
branch_asm.stub(branch_stub) do
case branch_stub.shape
in Default
2023-02-10 23:36:48 -08:00
branch_asm.public_send(opcode, branch_stub.target0.address)
2023-02-08 00:47:01 -08:00
end
end
end
branch_stub.compile.call(asm)
else
2023-02-08 01:48:32 -08:00
asm.public_send(opcode, side_exit)
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-04 21:26:46 -08:00
def jit_guard_known_klass(jit, ctx, asm, known_klass, obj_opnd, comptime_obj, side_exit, limit: 10)
# Only memory operand is supported for now
assert_equal(true, obj_opnd.is_a?(Array))
2023-03-17 23:34:36 -07:00
# Touching this as Ruby could crash for FrozenCore
known_klass = C.to_value(known_klass)
if known_klass == C.rb_cNilClass
asm.comment('guard object is nil')
asm.cmp(obj_opnd, Qnil)
jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)
2023-03-17 23:34:36 -07:00
elsif known_klass == C.rb_cTrueClass
asm.comment('guard object is true')
asm.cmp(obj_opnd, Qtrue)
jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)
2023-03-17 23:34:36 -07:00
elsif known_klass == C.rb_cFalseClass
asm.comment('guard object is false')
asm.cmp(obj_opnd, Qfalse)
jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)
2023-03-17 23:34:36 -07:00
elsif known_klass == C.rb_cInteger && fixnum?(comptime_obj)
asm.comment('guard object is fixnum')
asm.test(obj_opnd, C::RUBY_FIXNUM_FLAG)
jit_chain_guard(:jz, jit, ctx, asm, side_exit, limit:)
2023-03-17 23:34:36 -07:00
elsif known_klass == C.rb_cSymbol && static_symbol?(comptime_obj)
2023-02-16 22:29:58 -08:00
# We will guard STATIC vs DYNAMIC as though they were separate classes
# DYNAMIC symbols can be handled by the general else case below
asm.comment('guard object is static symbol')
assert_equal(8, C::RUBY_SPECIAL_SHIFT)
asm.cmp(BytePtr[*obj_opnd], C::RUBY_SYMBOL_FLAG)
2023-02-16 22:29:58 -08:00
jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)
2023-03-17 23:34:36 -07:00
elsif known_klass == C.rb_cFloat && flonum?(comptime_obj)
2023-03-01 23:23:02 -08:00
# We will guard flonum vs heap float as though they were separate classes
asm.comment('guard object is flonum')
asm.mov(:rax, obj_opnd)
asm.and(:rax, C::RUBY_FLONUM_MASK)
asm.cmp(:rax, C::RUBY_FLONUM_FLAG)
2023-03-01 23:23:02 -08:00
jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)
2023-03-17 23:34:36 -07:00
elsif C.FL_TEST(known_klass, C::RUBY_FL_SINGLETON) && comptime_obj == C.rb_class_attached_object(known_klass)
2023-02-08 01:48:32 -08:00
asm.comment('guard known object with singleton class')
2023-03-17 23:34:36 -07:00
asm.mov(:rax, to_value(comptime_obj))
2023-02-08 01:48:32 -08:00
asm.cmp(obj_opnd, :rax)
jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)
else
# Load memory to a register
asm.mov(:rax, obj_opnd)
obj_opnd = :rax
2023-02-08 01:48:32 -08:00
# Check that the receiver is a heap object
# Note: if we get here, the class doesn't have immediate instances.
asm.comment('guard not immediate')
asm.test(obj_opnd, C::RUBY_IMMEDIATE_MASK)
2023-02-08 01:48:32 -08:00
jit_chain_guard(:jnz, jit, ctx, asm, side_exit, limit:)
asm.cmp(obj_opnd, Qfalse)
jit_chain_guard(:je, jit, ctx, asm, side_exit, limit:)
# Bail if receiver class is different from known_klass
klass_opnd = [obj_opnd, C.RBasic.offsetof(:klass)]
2023-02-16 22:29:58 -08:00
asm.comment("guard known class #{known_klass}")
2023-03-17 23:34:36 -07:00
asm.mov(:rcx, known_klass)
2023-02-08 01:48:32 -08:00
asm.cmp(klass_opnd, :rcx)
jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)
2023-02-08 00:47:01 -08:00
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
2023-02-13 22:36:02 -08:00
def two_fixnums_on_stack?(jit)
comptime_recv = jit.peek_at_stack(1)
comptime_arg = jit.peek_at_stack(0)
return fixnum?(comptime_recv) && fixnum?(comptime_arg)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 22:36:02 -08:00
def guard_two_fixnums(jit, ctx, asm, side_exit)
# Get stack operands without popping them
arg1 = ctx.stack_opnd(0)
arg0 = ctx.stack_opnd(1)
asm.comment('guard arg0 fixnum')
asm.test(arg0, C::RUBY_FIXNUM_FLAG)
2023-02-13 22:36:02 -08:00
jit_chain_guard(:jz, jit, ctx, asm, side_exit)
# TODO: upgrade type, and skip the check when possible
asm.comment('guard arg1 fixnum')
asm.test(arg1, C::RUBY_FIXNUM_FLAG)
2023-02-13 22:36:02 -08:00
jit_chain_guard(:jz, jit, ctx, asm, side_exit)
# TODO: upgrade type, and skip the check when possible
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 21:48:24 -08:00
def jit_fixnum_cmp(jit, ctx, asm, opcode:, bop:)
opcode => :cmovl | :cmovle | :cmovg | :cmovge
unless jit.at_current_insn?
defer_compilation(jit, ctx, asm)
return EndBlock
end
comptime_recv = jit.peek_at_stack(1)
comptime_obj = jit.peek_at_stack(0)
if fixnum?(comptime_recv) && fixnum?(comptime_obj)
# Generate a side exit before popping operands
side_exit = side_exit(jit, ctx)
unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, bop)
2023-02-13 21:48:24 -08:00
return CantCompile
end
obj_opnd = ctx.stack_pop
recv_opnd = ctx.stack_pop
asm.comment('guard recv is fixnum') # TODO: skip this with type information
asm.test(recv_opnd, C::RUBY_FIXNUM_FLAG)
2023-02-13 21:48:24 -08:00
asm.jz(side_exit)
asm.comment('guard obj is fixnum') # TODO: skip this with type information
asm.test(obj_opnd, C::RUBY_FIXNUM_FLAG)
2023-02-13 21:48:24 -08:00
asm.jz(side_exit)
asm.mov(:rax, obj_opnd)
asm.cmp(recv_opnd, :rax)
asm.mov(:rax, Qfalse)
asm.mov(:rcx, Qtrue)
asm.public_send(opcode, :rax, :rcx)
dst_opnd = ctx.stack_push
asm.mov(dst_opnd, :rax)
KeepCompiling
else
opt_send_without_block(jit, ctx, asm)
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-02 23:00:52 -08:00
def jit_equality_specialized(jit, ctx, asm, gen_eq)
2023-02-13 23:57:40 -08:00
# Create a side-exit to fall back to the interpreter
side_exit = side_exit(jit, ctx)
a_opnd = ctx.stack_opnd(1)
b_opnd = ctx.stack_opnd(0)
comptime_a = jit.peek_at_stack(1)
comptime_b = jit.peek_at_stack(0)
if two_fixnums_on_stack?(jit)
unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_EQ)
2023-02-13 23:57:40 -08:00
return false
end
guard_two_fixnums(jit, ctx, asm, side_exit)
asm.comment('check fixnum equality')
asm.mov(:rax, a_opnd)
asm.mov(:rcx, b_opnd)
asm.cmp(:rax, :rcx)
2023-03-02 23:00:52 -08:00
asm.mov(:rax, gen_eq ? Qfalse : Qtrue)
asm.mov(:rcx, gen_eq ? Qtrue : Qfalse)
2023-02-13 23:57:40 -08:00
asm.cmove(:rax, :rcx)
# Push the output on the stack
ctx.stack_pop(2)
dst = ctx.stack_push
asm.mov(dst, :rax)
true
2023-03-03 22:30:18 -08:00
elsif C.rb_class_of(comptime_a) == String && C.rb_class_of(comptime_b) == String
unless Invariants.assume_bop_not_redefined(jit, C::STRING_REDEFINED_OP_FLAG, C::BOP_EQ)
2023-02-13 23:57:40 -08:00
# if overridden, emit the generic version
return false
end
# Guard that a is a String
2023-03-03 22:03:23 -08:00
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_a), a_opnd, comptime_a, side_exit)
2023-02-13 23:57:40 -08:00
equal_label = asm.new_label(:equal)
ret_label = asm.new_label(:ret)
# If they are equal by identity, return true
asm.mov(:rax, a_opnd)
asm.mov(:rcx, b_opnd)
asm.cmp(:rax, :rcx)
asm.je(equal_label)
# Otherwise guard that b is a T_STRING (from type info) or String (from runtime guard)
# Note: any T_STRING is valid here, but we check for a ::String for simplicity
# To pass a mutable static variable (rb_cString) requires an unsafe block
2023-03-03 22:03:23 -08:00
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_b), b_opnd, comptime_b, side_exit)
2023-02-13 23:57:40 -08:00
asm.comment('call rb_str_eql_internal')
asm.mov(C_ARGS[0], a_opnd)
asm.mov(C_ARGS[1], b_opnd)
asm.call(gen_eq ? C.rb_str_eql_internal : C.rjit_str_neq_internal)
2023-02-13 23:57:40 -08:00
# Push the output on the stack
ctx.stack_pop(2)
dst = ctx.stack_push
asm.mov(dst, C_RET)
asm.jmp(ret_label)
asm.write_label(equal_label)
2023-03-02 23:00:52 -08:00
asm.mov(dst, gen_eq ? Qtrue : Qfalse)
2023-02-13 23:57:40 -08:00
asm.write_label(ret_label)
true
else
false
end
end
2023-03-04 21:35:45 -08:00
# NOTE: This clobbers :rax
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 11:10:04 -08:00
def jit_prepare_routine_call(jit, ctx, asm)
2023-02-16 22:29:58 -08:00
jit.record_boundary_patch_point = true
2023-02-08 11:10:04 -08:00
jit_save_pc(jit, asm)
2023-03-17 23:27:16 -07:00
jit_save_sp(ctx, asm)
2023-02-08 11:10:04 -08:00
end
2023-03-17 23:27:16 -07:00
# NOTE: This clobbers :rax
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-09 16:25:06 -08:00
def jit_save_pc(jit, asm, comment: 'save PC to CFP')
2023-02-08 11:10:04 -08:00
next_pc = jit.pc + jit.insn.len * C.VALUE.size # Use the next one for backtrace and side exits
2023-02-09 16:25:06 -08:00
asm.comment(comment)
2023-02-08 11:10:04 -08:00
asm.mov(:rax, next_pc)
asm.mov([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax)
end
2023-03-06 23:15:30 -08:00
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-17 23:27:16 -07:00
def jit_save_sp(ctx, asm)
2023-02-08 11:10:04 -08:00
if ctx.sp_offset != 0
asm.comment('save SP to CFP')
2023-02-09 16:25:06 -08:00
asm.lea(SP, ctx.sp_opnd)
2023-02-08 11:10:04 -08:00
asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP)
ctx.sp_offset = 0
end
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 00:47:01 -08:00
def jump_to_next_insn(jit, ctx, asm)
reset_depth = ctx.dup
reset_depth.chain_depth = 0
next_pc = jit.pc + jit.insn.len * C.VALUE.size
2023-02-16 22:29:58 -08:00
# We are at the end of the current instruction. Record the boundary.
if jit.record_boundary_patch_point
exit_pos = Assembler.new.then do |ocb_asm|
@exit_compiler.compile_side_exit(next_pc, ctx, ocb_asm)
@ocb.write(ocb_asm)
end
Invariants.record_global_inval_patch(asm, exit_pos)
jit.record_boundary_patch_point = false
end
jit_direct_jump(jit.iseq, next_pc, reset_depth, asm, comment: 'jump_to_next_insn')
2023-02-08 00:47:01 -08:00
end
2023-02-07 14:42:58 -08:00
# rb_vm_check_ints
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-07 14:42:58 -08:00
def jit_check_ints(jit, ctx, asm)
2023-01-07 21:24:30 -08:00
asm.comment('RUBY_VM_CHECK_INTS(ec)')
asm.mov(:eax, DwordPtr[EC, C.rb_execution_context_t.offsetof(:interrupt_flag)])
2023-01-07 21:24:30 -08:00
asm.test(:eax, :eax)
asm.jnz(side_exit(jit, ctx))
end
2023-02-24 14:48:02 -08:00
# See get_lvar_level in compile.c
def get_lvar_level(iseq)
level = 0
while iseq.to_i != iseq.body.local_iseq.to_i
level += 1
iseq = iseq.body.parent_iseq
end
return level
end
# GET_LEP
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-24 14:48:02 -08:00
def jit_get_lep(jit, asm, reg:)
level = get_lvar_level(jit.iseq)
jit_get_ep(asm, level, reg:)
end
# vm_get_ep
2023-03-06 23:15:30 -08:00
# @param asm [RubyVM::RJIT::Assembler]
2023-02-13 22:50:52 -08:00
def jit_get_ep(asm, level, reg:)
asm.mov(reg, [CFP, C.rb_control_frame_t.offsetof(:ep)])
2023-02-08 01:17:39 -08:00
level.times do
# GET_PREV_EP: ep[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03
asm.mov(reg, [reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL])
2023-02-13 22:50:52 -08:00
asm.and(reg, ~0x03)
2023-02-08 01:17:39 -08:00
end
end
2023-02-07 14:42:58 -08:00
# vm_getivar
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-08 14:36:55 -08:00
def jit_getivar(jit, ctx, asm, comptime_obj, ivar_id, obj_opnd = nil)
2023-02-07 14:42:58 -08:00
side_exit = side_exit(jit, ctx)
2023-02-08 00:47:01 -08:00
starting_ctx = ctx.dup # copy for jit_chain_guard
2023-02-07 14:42:58 -08:00
# Guard not special const
if C::SPECIAL_CONST_P(comptime_obj)
2023-02-07 14:42:58 -08:00
asm.incr_counter(:getivar_special_const)
return CantCompile
end
case C::BUILTIN_TYPE(comptime_obj)
when C::T_OBJECT
2023-02-07 23:25:33 -08:00
# This is the only supported case for now (ROBJECT_IVPTR)
2023-02-07 14:42:58 -08:00
else
2023-03-03 20:59:58 -08:00
# General case. Call rb_ivar_get().
# VALUE rb_ivar_get(VALUE obj, ID id)
asm.comment('call rb_ivar_get()')
asm.mov(C_ARGS[0], obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)])
asm.mov(C_ARGS[1], ivar_id)
# The function could raise exceptions.
jit_prepare_routine_call(jit, ctx, asm) # clobbers obj_opnd and :rax
asm.call(C.rb_ivar_get)
if obj_opnd # attr_reader
ctx.stack_pop
end
# Push the ivar on the stack
out_opnd = ctx.stack_push
asm.mov(out_opnd, C_RET)
# Jump to next instruction. This allows guard chains to share the same successor.
jump_to_next_insn(jit, ctx, asm)
return EndBlock
2023-02-07 14:42:58 -08:00
end
2023-03-03 20:59:58 -08:00
asm.mov(:rax, obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)])
guard_object_is_heap(asm, :rax, counted_exit(side_exit, :getivar_not_heap))
2023-02-07 14:42:58 -08:00
shape_id = C.rb_shape_get_shape_id(comptime_obj)
if shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID
2023-02-07 14:42:58 -08:00
asm.incr_counter(:getivar_too_complex)
return CantCompile
end
asm.comment('guard shape')
asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id)
2023-02-08 00:47:01 -08:00
jit_chain_guard(:jne, jit, starting_ctx, asm, counted_exit(side_exit, :getivar_megamorphic))
2023-02-07 14:42:58 -08:00
index = C.rb_shape_get_iv_index(shape_id, ivar_id)
if index
2023-02-08 14:36:55 -08:00
asm.comment('ROBJECT_IVPTR')
if C::FL_TEST_RAW(comptime_obj, C::ROBJECT_EMBED)
2023-02-07 23:25:33 -08:00
# Access embedded array
2023-02-07 14:42:58 -08:00
asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :ary) + (index * C.VALUE.size)])
else
2023-02-07 23:25:33 -08:00
# Pull out an ivar table on heap
asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :heap, :ivptr)])
# Read the table
asm.mov(:rax, [:rax, index * C.VALUE.size])
2023-02-07 14:42:58 -08:00
end
2023-02-07 23:25:33 -08:00
val_opnd = :rax
2023-02-07 14:42:58 -08:00
else
val_opnd = Qnil
end
2023-02-08 14:36:55 -08:00
if obj_opnd
ctx.stack_pop # pop receiver for attr_reader
end
2023-02-07 14:42:58 -08:00
stack_opnd = ctx.stack_push
asm.mov(stack_opnd, val_opnd)
2023-02-08 00:47:01 -08:00
# Let guard chains share the same successor
jump_to_next_insn(jit, ctx, asm)
EndBlock
2023-02-07 14:42:58 -08:00
end
2023-03-02 23:44:57 -08:00
def jit_write_iv(asm, comptime_receiver, recv_reg, temp_reg, ivar_index, set_value, needs_extension)
2023-03-02 21:14:26 -08:00
# Compile time self is embedded and the ivar index lands within the object
embed_test_result = C::FL_TEST_RAW(comptime_receiver, C::ROBJECT_EMBED) && !needs_extension
2023-03-02 21:14:26 -08:00
if embed_test_result
# Find the IV offset
offs = C.RObject.offsetof(:as, :ary) + ivar_index * C.VALUE.size
# Write the IV
asm.comment('write IV')
asm.mov(temp_reg, set_value)
asm.mov([recv_reg, offs], temp_reg)
else
# Compile time value is *not* embedded.
# Get a pointer to the extended table
asm.mov(recv_reg, [recv_reg, C.RObject.offsetof(:as, :heap, :ivptr)])
# Write the ivar in to the extended table
asm.comment("write IV");
asm.mov(temp_reg, set_value)
asm.mov([recv_reg, C.VALUE.size * ivar_index], temp_reg)
end
end
2023-02-24 14:48:02 -08:00
# vm_caller_setup_arg_block
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-24 14:48:02 -08:00
def jit_caller_setup_arg_block(jit, ctx, asm, ci, blockiseq, is_super)
2023-03-04 13:39:04 -08:00
side_exit = side_exit(jit, ctx)
if C.vm_ci_flag(ci) & C::VM_CALL_ARGS_BLOCKARG != 0
2023-03-04 21:13:01 -08:00
# TODO: Skip cmp + jne using Context?
2023-03-04 13:39:04 -08:00
block_code = jit.peek_at_stack(0)
block_opnd = ctx.stack_opnd(0) # to be popped after eliminating side exit possibility
if block_code.nil?
asm.cmp(block_opnd, Qnil)
2023-03-04 21:26:46 -08:00
jit_chain_guard(:jne, jit, ctx, asm, counted_exit(side_exit, :send_block_not_nil))
return C::VM_BLOCK_HANDLER_NONE
2023-03-04 21:13:01 -08:00
elsif C.to_value(block_code) == C.rb_block_param_proxy
asm.mov(:rax, C.rb_block_param_proxy)
asm.cmp(block_opnd, :rax)
2023-03-04 21:26:46 -08:00
jit_chain_guard(:jne, jit, ctx, asm, counted_exit(side_exit, :send_block_not_proxy))
2023-03-04 21:13:01 -08:00
return C.rb_block_param_proxy
2023-03-04 13:39:04 -08:00
else
2023-03-04 21:13:01 -08:00
asm.incr_counter(:send_blockarg_not_nil_or_proxy)
2023-03-04 13:39:04 -08:00
return CantCompile
end
2023-02-24 14:48:02 -08:00
elsif blockiseq != 0
2023-02-28 23:22:12 -08:00
return blockiseq
2023-02-24 14:48:02 -08:00
else
if is_super
# GET_BLOCK_HANDLER();
# Guard no block passed. Only handle that case for now.
asm.comment('guard no block given')
jit_get_lep(jit, asm, reg: :rax)
asm.cmp([:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL], C::VM_BLOCK_HANDLER_NONE)
2023-03-04 13:39:04 -08:00
asm.jne(counted_exit(side_exit, :send_block_handler))
return C::VM_BLOCK_HANDLER_NONE
2023-02-24 14:48:02 -08:00
else
2023-02-28 23:22:12 -08:00
# Not implemented yet. Is this even necessary?
2023-02-28 22:50:04 -08:00
asm.incr_counter(:send_block_setup)
return CantCompile
2023-02-24 14:48:02 -08:00
end
end
2023-02-17 20:57:16 -08:00
end
2023-01-07 21:24:30 -08:00
2023-02-24 14:48:02 -08:00
# vm_search_method
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_search_method(jit, ctx, asm, mid, calling)
2023-02-24 14:48:02 -08:00
assert_equal(true, jit.at_current_insn?)
2023-01-07 21:24:30 -08:00
2023-02-08 01:48:32 -08:00
# Generate a side exit
side_exit = side_exit(jit, ctx)
2023-02-16 22:29:58 -08:00
# kw_splat is not supported yet
if calling.flags & C::VM_CALL_KW_SPLAT != 0
2023-02-06 15:07:58 -08:00
asm.incr_counter(:send_kw_splat)
2023-02-03 22:42:13 -08:00
return CantCompile
end
2023-01-07 21:24:30 -08:00
2023-02-08 01:48:32 -08:00
# Get a compile-time receiver and its class
recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet
recv_idx += calling.send_shift
comptime_recv = jit.peek_at_stack(recv_idx)
2023-01-07 21:24:30 -08:00
comptime_recv_klass = C.rb_class_of(comptime_recv)
2023-02-03 22:42:13 -08:00
# Guard the receiver class (part of vm_search_method_fastpath)
2023-02-16 22:29:58 -08:00
recv_opnd = ctx.stack_opnd(recv_idx)
2023-02-08 01:48:32 -08:00
megamorphic_exit = counted_exit(side_exit, :send_klass_megamorphic)
2023-03-01 23:23:02 -08:00
jit_guard_known_klass(jit, ctx, asm, comptime_recv_klass, recv_opnd, comptime_recv, megamorphic_exit)
2023-01-07 21:24:30 -08:00
2023-02-03 22:42:13 -08:00
# Do method lookup (vm_cc_cme(cc) != NULL)
2023-01-07 21:24:30 -08:00
cme = C.rb_callable_method_entry(comptime_recv_klass, mid)
if cme.nil?
2023-02-06 15:07:58 -08:00
asm.incr_counter(:send_missing_cme)
2023-02-03 22:42:13 -08:00
return CantCompile # We don't support vm_call_method_name
2023-01-07 21:24:30 -08:00
end
2023-02-24 14:48:02 -08:00
# Invalidate on redefinition (part of vm_search_method_fastpath)
Invariants.assume_method_lookup_stable(jit, cme)
2023-03-03 23:51:31 -08:00
return cme, comptime_recv_klass
2023-02-24 14:48:02 -08:00
end
def jit_search_super_method(jit, ctx, asm, mid, calling)
2023-02-24 14:48:02 -08:00
assert_equal(true, jit.at_current_insn?)
me = C.rb_vm_frame_method_entry(jit.cfp)
if me.nil?
return CantCompile
end
# FIXME: We should track and invalidate this block when this cme is invalidated
current_defined_class = me.defined_class
mid = me.def.original_id
if me.to_i != C.rb_callable_method_entry(current_defined_class, me.called_id).to_i
# Though we likely could generate this call, as we are only concerned
# with the method entry remaining valid, assume_method_lookup_stable
# below requires that the method lookup matches as well
return CantCompile
end
# vm_search_normal_superclass
rbasic_klass = C.to_ruby(C.RBasic.new(C.to_value(current_defined_class)).klass)
if C::BUILTIN_TYPE(current_defined_class) == C::RUBY_T_ICLASS && C::BUILTIN_TYPE(rbasic_klass) == C::RUBY_T_MODULE && \
C::FL_TEST_RAW(rbasic_klass, C::RMODULE_IS_REFINEMENT) != 0
2023-02-24 14:48:02 -08:00
return CantCompile
end
2023-03-21 00:10:14 -07:00
comptime_superclass = C.rb_class_get_superclass(C.RCLASS_ORIGIN(current_defined_class))
2023-02-24 14:48:02 -08:00
# Don't JIT calls that aren't simple
# Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block.
if calling.flags & C::VM_CALL_KWARG != 0
2023-02-24 14:48:02 -08:00
asm.incr_counter(:send_kwarg)
return CantCompile
end
if calling.flags & C::VM_CALL_KW_SPLAT != 0
2023-02-24 14:48:02 -08:00
asm.incr_counter(:send_kw_splat)
return CantCompile
end
# Ensure we haven't rebound this method onto an incompatible class.
# In the interpreter we try to avoid making this check by performing some
# cheaper calculations first, but since we specialize on the method entry
# and so only have to do this once at compile time this is fine to always
# check and side exit.
comptime_recv = jit.peek_at_stack(calling.argc)
2023-03-03 21:50:49 -08:00
unless C.obj_is_kind_of(comptime_recv, current_defined_class)
2023-02-24 14:48:02 -08:00
return CantCompile
end
# Do method lookup
cme = C.rb_callable_method_entry(comptime_superclass, mid)
if cme.nil?
return CantCompile
end
# Check that we'll be able to write this method dispatch before generating checks
cme_def_type = cme.def.type
if cme_def_type != C::VM_METHOD_TYPE_ISEQ && cme_def_type != C::VM_METHOD_TYPE_CFUNC
2023-02-24 14:48:02 -08:00
# others unimplemented
return CantCompile
end
# Guard that the receiver has the same class as the one from compile time
side_exit = side_exit(jit, ctx)
asm.comment('guard known me')
jit_get_lep(jit, asm, reg: :rax)
asm.mov(:rcx, me.to_i)
asm.cmp([:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_ME_CREF], :rcx)
2023-02-24 14:48:02 -08:00
asm.jne(counted_exit(side_exit, :invokesuper_me_changed))
# We need to assume that both our current method entry and the super
# method entry we invoke remain stable
Invariants.assume_method_lookup_stable(jit, me)
Invariants.assume_method_lookup_stable(jit, cme)
return cme
end
# vm_call_general
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_general(jit, ctx, asm, mid, calling, cme, known_recv_class)
jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class)
2023-02-24 14:48:02 -08:00
end
# vm_call_method
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-24 14:48:02 -08:00
# @param send_shift [Integer] The number of shifts needed for VM_CALL_OPT_SEND
def jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class)
2023-02-03 22:42:13 -08:00
# The main check of vm_call_method before vm_call_method_each_type
case C::METHOD_ENTRY_VISI(cme)
in C::METHOD_VISI_PUBLIC
2023-01-07 21:24:30 -08:00
# You can always call public methods
in C::METHOD_VISI_PRIVATE
2023-02-03 22:42:13 -08:00
# Allow only callsites without a receiver
if calling.flags & C::VM_CALL_FCALL == 0
2023-02-06 15:07:58 -08:00
asm.incr_counter(:send_private)
2023-01-07 21:24:30 -08:00
return CantCompile
end
in C::METHOD_VISI_PROTECTED
2023-03-03 21:50:49 -08:00
# If the method call is an FCALL, it is always valid
if calling.flags & C::VM_CALL_FCALL == 0
2023-03-03 21:50:49 -08:00
# otherwise we need an ancestry check to ensure the receiver is valid to be called as protected
jit_protected_callee_ancestry_guard(asm, cme, side_exit(jit, ctx))
end
2023-01-07 21:24:30 -08:00
end
2023-02-24 14:48:02 -08:00
# Get a compile-time receiver
recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet
recv_idx += calling.send_shift
comptime_recv = jit.peek_at_stack(recv_idx)
2023-02-24 14:48:02 -08:00
recv_opnd = ctx.stack_opnd(recv_idx)
2023-01-07 21:24:30 -08:00
jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)
2023-02-03 22:42:13 -08:00
end
2023-01-07 21:24:30 -08:00
2023-03-03 21:50:49 -08:00
# Generate ancestry guard for protected callee.
# Calls to protected callees only go through when self.is_a?(klass_that_defines_the_callee).
def jit_protected_callee_ancestry_guard(asm, cme, side_exit)
# See vm_call_method().
def_class = cme.defined_class
# Note: PC isn't written to current control frame as rb_is_kind_of() shouldn't raise.
# VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass);
asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)])
asm.mov(C_ARGS[1], to_value(def_class))
asm.call(C.rb_obj_is_kind_of)
asm.test(C_RET, C_RET)
asm.jz(counted_exit(side_exit, :send_protected_check_failed))
end
2023-02-03 22:42:13 -08:00
# vm_call_method_each_type
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)
2023-01-07 21:24:30 -08:00
case cme.def.type
in C::VM_METHOD_TYPE_ISEQ
2023-03-04 22:42:03 -08:00
iseq = def_iseq_ptr(cme.def)
jit_call_iseq(jit, ctx, asm, cme, calling, iseq)
in C::VM_METHOD_TYPE_NOTIMPLEMENTED
2023-02-15 23:43:53 -08:00
asm.incr_counter(:send_notimplemented)
return CantCompile
in C::VM_METHOD_TYPE_CFUNC
jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class)
in C::VM_METHOD_TYPE_ATTRSET
jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd)
in C::VM_METHOD_TYPE_IVAR
jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd)
in C::VM_METHOD_TYPE_MISSING
2023-02-15 23:43:53 -08:00
asm.incr_counter(:send_missing)
return CantCompile
in C::VM_METHOD_TYPE_BMETHOD
jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)
in C::VM_METHOD_TYPE_ALIAS
jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)
in C::VM_METHOD_TYPE_OPTIMIZED
jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class)
in C::VM_METHOD_TYPE_UNDEF
2023-02-15 23:43:53 -08:00
asm.incr_counter(:send_undef)
2023-02-08 14:24:10 -08:00
return CantCompile
in C::VM_METHOD_TYPE_ZSUPER
2023-02-08 14:24:10 -08:00
asm.incr_counter(:send_zsuper)
return CantCompile
in C::VM_METHOD_TYPE_REFINED
2023-02-08 14:24:10 -08:00
asm.incr_counter(:send_refined)
return CantCompile
2023-01-07 21:24:30 -08:00
end
end
# vm_call_iseq_setup
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type: nil, prev_ep: nil)
argc = calling.argc
flags = calling.flags
send_shift = calling.send_shift
block_handler = calling.block_handler
# When you have keyword arguments, there is an extra object that gets
# placed on the stack the represents a bitmap of the keywords that were not
# specified at the call site. We need to keep track of the fact that this
# value is present on the stack in order to properly set up the callee's
# stack pointer.
doing_kw_call = iseq.body.param.flags.has_kw
supplying_kws = flags & C::VM_CALL_KWARG != 0
if flags & C::VM_CALL_TAILCALL != 0
# We can't handle tailcalls
asm.incr_counter(:send_tailcall)
return CantCompile
end
# No support for callees with these parameters yet as they require allocation
# or complex handling.
if iseq.body.param.flags.has_post
2023-04-01 17:24:06 -07:00
asm.incr_counter(:send_iseq_has_opt)
return CantCompile
end
if iseq.body.param.flags.has_kwrest
2023-04-01 17:24:06 -07:00
asm.incr_counter(:send_iseq_has_kwrest)
return CantCompile
end
# In order to handle backwards compatibility between ruby 3 and 2
# ruby2_keywords was introduced. It is called only on methods
# with splat and changes they way they handle them.
# We are just going to not compile these.
# https://www.rubydoc.info/stdlib/core/Proc:ruby2_keywords
if iseq.body.param.flags.ruby2_keywords && flags & C::VM_CALL_ARGS_SPLAT != 0
asm.incr_counter(:send_iseq_ruby2_keywords)
return CantCompile
end
iseq_has_rest = iseq.body.param.flags.has_rest
if iseq_has_rest && block_handler == :captured
asm.incr_counter(:send_iseq_has_rest_and_captured)
return CantCompile
end
if iseq_has_rest && iseq.body.param.flags.has_kw
asm.incr_counter(:send_iseq_has_rest_and_kw)
return CantCompile
end
# If we have keyword arguments being passed to a callee that only takes
# positionals, then we need to allocate a hash. For now we're going to
# call that too complex and bail.
if supplying_kws && !iseq.body.param.flags.has_kw
asm.incr_counter(:send_iseq_has_no_kw)
return CantCompile
end
# If we have a method accepting no kwargs (**nil), exit if we have passed
# it any kwargs.
if supplying_kws && iseq.body.param.flags.accepts_no_kwarg
2023-04-01 17:24:06 -07:00
asm.incr_counter(:send_iseq_accepts_no_kwarg)
return CantCompile
end
# For computing number of locals to set up for the callee
num_params = iseq.body.param.size
# Block parameter handling. This mirrors setup_parameters_complex().
if iseq.body.param.flags.has_block
if iseq.body.local_iseq.to_i == iseq.to_i
num_params -= 1
else
# In this case (param.flags.has_block && local_iseq != iseq),
# the block argument is setup as a local variable and requires
# materialization (allocation). Bail.
asm.incr_counter(:send_iseq_materialized_block)
return CantCompile
end
end
if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_ZSUPER != 0
# zsuper methods are super calls without any arguments.
# They are also marked as splat, but don't actually have an array
# they pull arguments from, instead we need to change to call
# a different method with the current stack.
asm.incr_counter(:send_iseq_zsuper)
return CantCompile
end
start_pc_offset = 0
required_num = iseq.body.param.lead_num
# This struct represents the metadata about the caller-specified
# keyword arguments.
kw_arg = calling.kwarg
kw_arg_num = if kw_arg.nil?
0
else
kw_arg.keyword_len
end
# Arity handling and optional parameter setup
opts_filled = argc - required_num - kw_arg_num
opt_num = iseq.body.param.opt_num
opts_missing = opt_num - opts_filled
if doing_kw_call && flags & C::VM_CALL_ARGS_SPLAT != 0
asm.incr_counter(:send_iseq_splat_with_kw)
return CantCompile
end
if iseq_has_rest && opt_num != 0
asm.incr_counter(:send_iseq_has_rest_and_optional)
return CantCompile
end
if opts_filled < 0 && flags & C::VM_CALL_ARGS_SPLAT == 0
# Too few arguments and no splat to make up for it
asm.incr_counter(:send_iseq_arity_error)
return CantCompile
end
if opts_filled > opt_num && !iseq_has_rest
# Too many arguments and no place to put them (i.e. rest arg)
asm.incr_counter(:send_iseq_arity_error)
return CantCompile
end
# block_arg = flags & C::VM_CALL_ARGS_BLOCKARG != 0
# jit_caller_setup_arg_block already handled send_blockarg_not_nil_or_proxy
# If we have unfilled optional arguments and keyword arguments then we
# would need to adjust the arguments location to account for that.
# For now we aren't handling this case.
if doing_kw_call && opts_missing > 0
asm.incr_counter(:send_iseq_missing_optional_kw)
return CantCompile
end
# We will handle splat case later
if opt_num > 0 && flags & C::VM_CALL_ARGS_SPLAT == 0
num_params -= opts_missing
start_pc_offset = iseq.body.param.opt_table[opts_filled]
end
if doing_kw_call
asm.incr_counter(:send_iseq_kw_call)
return CantCompile
end
# Number of locals that are not parameters
num_locals = iseq.body.local_table_size - num_params
# Pop blockarg after all side exits
if flags & C::VM_CALL_ARGS_BLOCKARG != 0
ctx.stack_pop(1)
end
if block_handler == C::VM_BLOCK_HANDLER_NONE && iseq.body.builtin_attrs & C::BUILTIN_ATTR_LEAF != 0
if jit_leaf_builtin_func(jit, ctx, asm, flags, iseq)
return KeepCompiling
end
end
# Stack overflow check
# Note that vm_push_frame checks it against a decremented cfp, hence the multiply by 2.
# #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin)
asm.comment('stack overflow check')
locals_offs = C.VALUE.size * (num_locals + iseq.body.stack_max) + 2 * C.rb_control_frame_t.size
asm.lea(:rax, ctx.sp_opnd(locals_offs))
asm.cmp(CFP, :rax)
asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow))
# Check if we need the arg0 splat handling of vm_callee_setup_block_arg
arg_setup_block = (block_handler == :captured) # arg_setup_type: arg_setup_block (invokeblock)
block_arg0_splat = arg_setup_block && argc == 1 &&
iseq.body.param.flags.has_lead && !iseq.body.param.flags.ambiguous_param0
# push_splat_args does stack manipulation so we can no longer side exit
if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest
asm.incr_counter(:send_iseq_splat)
return CantCompile
end
# This is a .send call and we need to adjust the stack
if flags & C::VM_CALL_OPT_SEND != 0
jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:)
end
if doing_kw_call
asm.incr_counter(:send_iseq_kw_call)
return CantCompile
end
# Same as vm_callee_setup_block_arg_arg0_check and vm_callee_setup_block_arg_arg0_splat
# on vm_callee_setup_block_arg for arg_setup_block. This is done after CALLER_SETUP_ARG
# and CALLER_REMOVE_EMPTY_KW_SPLAT, so this implementation is put here. This may need
# side exits, so you still need to allow side exits here if block_arg0_splat is true.
# Note that you can't have side exits after this arg0 splat.
if block_arg0_splat
asm.incr_counter(:send_iseq_block_arg0_splat)
return CantCompile
end
if iseq_has_rest
asm.incr_counter(:send_iseq_has_rest)
return CantCompile
end
# Setup the new frame
frame_type ||= C::VM_FRAME_MAGIC_METHOD | C::VM_ENV_FLAG_LOCAL
jit_push_frame(
jit, ctx, asm, cme, flags, argc, frame_type, block_handler,
iseq: iseq,
local_size: num_locals,
stack_max: iseq.body.stack_max,
prev_ep:,
)
# Create a context for the callee
callee_ctx = Context.new
# Directly jump to the entry point of the callee
pc = (iseq.body.iseq_encoded + start_pc_offset).to_i
jit_direct_jump(iseq, pc, callee_ctx, asm)
EndBlock
end
2023-03-26 19:20:03 -07:00
def jit_leaf_builtin_func(jit, ctx, asm, flags, iseq)
builtin_func = builtin_function(iseq)
if builtin_func.nil?
return false
end
# this is a .send call not currently supported for builtins
if flags & C::VM_CALL_OPT_SEND != 0
return false
end
builtin_argc = builtin_func.argc
if builtin_argc + 1 >= C_ARGS.size
return false
end
asm.comment('inlined leaf builtin')
# Skip this if it doesn't trigger GC
if iseq.body.builtin_attrs & C::BUILTIN_ATTR_NO_GC == 0
# The callee may allocate, e.g. Integer#abs on a Bignum.
# Save SP for GC, save PC for allocation tracing, and prepare
# for global invalidation after GC's VM lock contention.
jit_prepare_routine_call(jit, ctx, asm)
end
# Call the builtin func (ec, recv, arg1, arg2, ...)
asm.mov(C_ARGS[0], EC)
# Copy self and arguments
(0..builtin_argc).each do |i|
stack_opnd = ctx.stack_opnd(builtin_argc - i)
asm.mov(C_ARGS[i + 1], stack_opnd)
end
ctx.stack_pop(builtin_argc + 1)
asm.call(builtin_func.func_ptr)
# Push the return value
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
return true
end
2023-02-09 16:25:06 -08:00
# vm_call_cfunc
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class)
argc = calling.argc
flags = calling.flags
send_shift = calling.send_shift
block_handler = calling.block_handler
2023-03-25 00:52:07 -07:00
if jit_caller_setup_arg(jit, ctx, asm, flags, splat: true) == CantCompile
2023-02-09 16:25:06 -08:00
return CantCompile
end
if jit_caller_remove_empty_kw_splat(jit, ctx, asm, flags) == CantCompile
return CantCompile
end
2023-03-03 23:51:31 -08:00
jit_call_cfunc_with_frame(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:)
2023-02-09 16:25:06 -08:00
end
# jit_call_cfunc_with_frame
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-03 23:51:31 -08:00
def jit_call_cfunc_with_frame(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:)
2023-02-09 16:25:06 -08:00
cfunc = cme.def.body.cfunc
if argc + 1 > C_ARGS.size
2023-02-09 16:25:06 -08:00
asm.incr_counter(:send_cfunc_too_many_args)
return CantCompile
end
frame_type = C::VM_FRAME_MAGIC_CFUNC | C::VM_FRAME_FLAG_CFRAME | C::VM_ENV_FLAG_LOCAL
if flags & C::VM_CALL_KW_SPLAT != 0
frame_type |= C::VM_FRAME_FLAG_CFRAME_KW
2023-02-09 16:25:06 -08:00
end
# EXEC_EVENT_HOOK: RUBY_EVENT_C_CALL and RUBY_EVENT_C_RETURN
2023-03-26 17:41:05 -07:00
if c_method_tracing_currently_enabled?
asm.incr_counter(:send_c_tracing)
return CantCompile
end
2023-02-09 16:25:06 -08:00
# rb_check_arity
2023-02-10 22:07:34 -08:00
if cfunc.argc >= 0 && argc != cfunc.argc
2023-02-09 16:25:06 -08:00
asm.incr_counter(:send_arity)
return CantCompile
end
2023-02-14 23:44:44 -08:00
if cfunc.argc == -2
asm.incr_counter(:send_cfunc_ruby_array_varg)
return CantCompile
end
2023-02-09 16:25:06 -08:00
2023-03-02 22:23:47 -08:00
# Delegate to codegen for C methods if we have it.
if flags & C::VM_CALL_KWARG == 0 && flags & C::VM_CALL_OPT_SEND == 0
2023-03-02 22:23:47 -08:00
known_cfunc_codegen = lookup_cfunc_codegen(cme.def)
2023-03-03 23:51:31 -08:00
if known_cfunc_codegen&.call(jit, ctx, asm, argc, known_recv_class)
2023-03-02 22:23:47 -08:00
# cfunc codegen generated code. Terminate the block so
# there isn't multiple calls in the same block.
jump_to_next_insn(jit, ctx, asm)
return EndBlock
end
end
2023-03-25 00:52:07 -07:00
# Push splat args, which was skipped in jit_caller_setup_arg.
if flags & C::VM_CALL_ARGS_SPLAT != 0
# We aren't handling a vararg cfuncs with splat currently.
if cfunc.argc == -1
asm.incr_counter(:send_args_splat_cfunc_var_args)
return CantCompile
end
2023-03-25 00:52:07 -07:00
required_args = cfunc.argc - (argc - 1)
# + 1 for self
if required_args + 1 >= C_ARGS.size
asm.incr_counter(:send_cfunc_too_many_args)
return CantCompile
end
# We are going to assume that the splat fills all the remaining arguments.
# So the number of args should just equal the number of args the cfunc takes.
# In the generated code we test if this is true and if not side exit.
argc = cfunc.argc
jit_caller_setup_arg_splat(jit, ctx, asm, required_args)
end
# We will not have side exits from here. Adjust the stack, which was skipped in jit_call_opt_send.
if flags & C::VM_CALL_OPT_SEND != 0
2023-02-16 22:29:58 -08:00
jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:)
end
2023-02-13 00:00:26 -08:00
# Check interrupts before SP motion to safely side-exit with the original SP.
jit_check_ints(jit, ctx, asm)
# Stack overflow check
# #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin)
# REG_CFP <= REG_SP + 4 * SIZEOF_VALUE + sizeof(rb_control_frame_t)
asm.comment('stack overflow check')
asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * 4 + 2 * C.rb_control_frame_t.size))
asm.cmp(CFP, :rax)
asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow))
# Pop blockarg after all side exits
if flags & C::VM_CALL_ARGS_BLOCKARG != 0
ctx.stack_pop(1)
end
2023-02-09 16:25:06 -08:00
# Push a callee frame. SP register and ctx are not modified inside this.
2023-02-28 23:22:12 -08:00
jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, block_handler)
2023-02-09 16:25:06 -08:00
asm.comment('call C function')
2023-02-10 22:07:34 -08:00
case cfunc.argc
in (0..) # Non-variadic method
# Push receiver and args
(1 + argc).times do |i|
asm.mov(C_ARGS[i], ctx.stack_opnd(argc - i)) # TODO: +1 for VM_CALL_ARGS_BLOCKARG
end
in -1 # Variadic method: rb_f_puts(int argc, VALUE *argv, VALUE recv)
asm.mov(C_ARGS[0], argc)
asm.lea(C_ARGS[1], ctx.stack_opnd(argc - 1)) # argv
asm.mov(C_ARGS[2], ctx.stack_opnd(argc)) # recv
2023-02-09 16:25:06 -08:00
end
asm.mov(:rax, cfunc.func)
asm.call(:rax) # TODO: use rel32 if close enough
ctx.stack_pop(1 + argc)
2023-03-09 21:55:14 -08:00
Invariants.record_global_inval_patch(asm, full_cfunc_return)
2023-02-10 11:43:53 -08:00
2023-02-09 16:25:06 -08:00
asm.comment('push the return value')
stack_ret = ctx.stack_push
2023-02-10 22:07:34 -08:00
asm.mov(stack_ret, C_RET)
2023-02-09 16:25:06 -08:00
asm.comment('pop the stack frame')
asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], CFP)
# Let guard chains share the same successor (ctx.sp_offset == 1)
assert_equal(1, ctx.sp_offset)
jump_to_next_insn(jit, ctx, asm)
EndBlock
end
2023-03-26 17:41:05 -07:00
# vm_call_attrset
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd)
argc = calling.argc
flags = calling.flags
send_shift = calling.send_shift
2023-03-26 17:41:05 -07:00
if flags & C::VM_CALL_ARGS_SPLAT != 0
asm.incr_counter(:send_attrset_splat)
return CantCompile
end
if flags & C::VM_CALL_KWARG != 0
asm.incr_counter(:send_attrset_kwarg)
return CantCompile
elsif argc != 1 || !C.RB_TYPE_P(comptime_recv, C::RUBY_T_OBJECT)
asm.incr_counter(:send_attrset_method)
return CantCompile
elsif c_method_tracing_currently_enabled?
# Can't generate code for firing c_call and c_return events
# See :attr-tracing:
asm.incr_counter(:send_c_tracingg)
return CantCompile
elsif flags & C::VM_CALL_ARGS_BLOCKARG != 0
asm.incr_counter(:send_attrset_blockarg)
return CantCompile
end
ivar_name = cme.def.body.attr.id
# This is a .send call and we need to adjust the stack
if flags & C::VM_CALL_OPT_SEND != 0
jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:)
end
# Save the PC and SP because the callee may allocate
# Note that this modifies REG_SP, which is why we do it first
jit_prepare_routine_call(jit, ctx, asm)
# Get the operands from the stack
val_opnd = ctx.stack_pop(1)
recv_opnd = ctx.stack_pop(1)
# Call rb_vm_set_ivar_id with the receiver, the ivar name, and the value
asm.mov(C_ARGS[0], recv_opnd)
asm.mov(C_ARGS[1], ivar_name)
asm.mov(C_ARGS[2], val_opnd)
asm.call(C.rb_vm_set_ivar_id)
out_opnd = ctx.stack_push
asm.mov(out_opnd, C_RET)
KeepCompiling
end
2023-02-16 22:29:58 -08:00
# vm_call_ivar (+ part of vm_call_method_each_type)
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd)
argc = calling.argc
flags = calling.flags
if flags & C::VM_CALL_ARGS_SPLAT != 0
2023-02-08 14:36:55 -08:00
asm.incr_counter(:send_ivar_splat)
return CantCompile
end
if argc != 0
2023-02-09 21:49:35 -08:00
asm.incr_counter(:send_arity)
2023-02-08 14:36:55 -08:00
return CantCompile
end
2023-02-16 22:29:58 -08:00
# We don't support jit_call_opt_send_shift_stack for this yet.
if flags & C::VM_CALL_OPT_SEND != 0
2023-02-08 14:36:55 -08:00
asm.incr_counter(:send_ivar_opt_send)
return CantCompile
end
ivar_id = cme.def.body.attr.id
2023-02-28 23:22:12 -08:00
# Not handling block_handler
if flags & C::VM_CALL_ARGS_BLOCKARG != 0
2023-02-08 14:36:55 -08:00
asm.incr_counter(:send_ivar_blockarg)
return CantCompile
end
jit_getivar(jit, ctx, asm, comptime_recv, ivar_id, recv_opnd)
end
2023-03-04 22:42:03 -08:00
# vm_call_bmethod
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)
2023-03-04 22:42:03 -08:00
proc_addr = cme.def.body.bmethod.proc
proc_t = C.rb_yjit_get_proc_ptr(proc_addr)
proc_block = proc_t.block
if proc_block.type != C.block_type_iseq
2023-03-04 23:20:54 -08:00
asm.incr_counter(:send_bmethod_not_iseq)
2023-03-04 22:42:03 -08:00
return CantCompile
end
capture = proc_block.as.captured
iseq = capture.code.iseq
# TODO: implement this
# Optimize for single ractor mode and avoid runtime check for
# "defined with an un-shareable Proc in a different Ractor"
# if !assume_single_ractor_mode(jit, ocb)
# return CantCompile;
# end
# Passing a block to a block needs logic different from passing
# a block to a method and sometimes requires allocation. Bail for now.
if calling.block_handler != C::VM_BLOCK_HANDLER_NONE
2023-03-04 23:20:54 -08:00
asm.incr_counter(:send_bmethod_blockarg)
2023-03-04 22:42:03 -08:00
return CantCompile
end
2023-04-01 16:59:19 -07:00
jit_call_iseq(
jit, ctx, asm, cme, calling, iseq,
frame_type: C::VM_FRAME_MAGIC_BLOCK | C::VM_FRAME_FLAG_BMETHOD | C::VM_FRAME_FLAG_LAMBDA,
prev_ep: capture.ep,
)
2023-03-04 22:42:03 -08:00
end
2023-03-04 22:22:56 -08:00
# vm_call_alias
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)
2023-03-04 22:22:56 -08:00
cme = C.rb_aliased_callable_method_entry(cme)
jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)
2023-03-04 22:22:56 -08:00
end
2023-02-15 23:43:53 -08:00
# vm_call_optimized
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class)
if calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0
2023-03-04 13:39:04 -08:00
# Not working yet
asm.incr_counter(:send_optimized_blockarg)
return CantCompile
end
2023-02-15 23:43:53 -08:00
case cme.def.body.optimized.type
in C::OPTIMIZED_METHOD_TYPE_SEND
jit_call_opt_send(jit, ctx, asm, cme, calling, known_recv_class)
in C::OPTIMIZED_METHOD_TYPE_CALL
jit_call_opt_call(jit, ctx, asm, cme, calling.flags, calling.argc, calling.block_handler, known_recv_class, send_shift: calling.send_shift)
in C::OPTIMIZED_METHOD_TYPE_BLOCK_CALL
2023-02-15 23:43:53 -08:00
asm.incr_counter(:send_optimized_block_call)
return CantCompile
in C::OPTIMIZED_METHOD_TYPE_STRUCT_AREF
jit_call_opt_struct_aref(jit, ctx, asm, cme, calling.flags, calling.argc, calling.block_handler, known_recv_class, send_shift: calling.send_shift)
in C::OPTIMIZED_METHOD_TYPE_STRUCT_ASET
2023-02-15 23:43:53 -08:00
asm.incr_counter(:send_optimized_struct_aset)
return CantCompile
end
end
2023-02-16 22:29:58 -08:00
# vm_call_opt_send
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_opt_send(jit, ctx, asm, cme, calling, known_recv_class)
if jit_caller_setup_arg(jit, ctx, asm, calling.flags) == CantCompile
2023-02-16 22:29:58 -08:00
return CantCompile
end
if calling.argc == 0
2023-02-16 22:29:58 -08:00
asm.incr_counter(:send_optimized_send_no_args)
return CantCompile
end
calling.argc -= 1
2023-02-16 22:29:58 -08:00
# We aren't handling `send(:send, ...)` yet. This might work, but not tested yet.
if calling.send_shift > 0
2023-02-16 22:29:58 -08:00
asm.incr_counter(:send_optimized_send_send)
return CantCompile
end
# Lazily handle stack shift in jit_call_opt_send_shift_stack
calling.send_shift += 1
2023-02-16 22:29:58 -08:00
jit_call_symbol(jit, ctx, asm, cme, calling, known_recv_class, C::VM_CALL_FCALL)
2023-02-16 22:29:58 -08:00
end
2023-03-04 21:35:45 -08:00
# vm_call_opt_call
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-04 21:35:45 -08:00
def jit_call_opt_call(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:)
if block_handler != C::VM_BLOCK_HANDLER_NONE
2023-03-04 21:35:45 -08:00
asm.incr_counter(:send_optimized_call_block)
return CantCompile
end
if flags & C::VM_CALL_KWARG != 0
2023-03-04 21:35:45 -08:00
asm.incr_counter(:send_optimized_call_kwarg)
return CantCompile
end
if flags & C::VM_CALL_ARGS_SPLAT != 0
2023-03-04 21:35:45 -08:00
asm.incr_counter(:send_optimized_call_splat)
return CantCompile
end
# TODO: implement this
# Optimize for single ractor mode and avoid runtime check for
# "defined with an un-shareable Proc in a different Ractor"
# if !assume_single_ractor_mode(jit, ocb)
# return CantCompile
# end
# If this is a .send call we need to adjust the stack
if flags & C::VM_CALL_OPT_SEND != 0
2023-03-04 21:35:45 -08:00
jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:)
end
# About to reset the SP, need to load this here
recv_idx = argc # blockarg is not supported. send_shift is already handled.
asm.mov(:rcx, ctx.stack_opnd(recv_idx)) # recv
# Save the PC and SP because the callee can make Ruby calls
jit_prepare_routine_call(jit, ctx, asm) # NOTE: clobbers rax
asm.lea(:rax, ctx.sp_opnd(0)) # sp
kw_splat = flags & C::VM_CALL_KW_SPLAT
2023-03-04 21:35:45 -08:00
asm.mov(C_ARGS[0], :rcx)
asm.mov(C_ARGS[1], EC)
asm.mov(C_ARGS[2], argc)
asm.lea(C_ARGS[3], [:rax, -argc * C.VALUE.size]) # stack_argument_pointer. NOTE: C_ARGS[3] is rcx
asm.mov(C_ARGS[4], kw_splat)
asm.mov(C_ARGS[5], C::VM_BLOCK_HANDLER_NONE)
asm.call(C.rjit_optimized_call)
2023-03-04 21:35:45 -08:00
ctx.stack_pop(argc + 1)
stack_ret = ctx.stack_push
asm.mov(stack_ret, C_RET)
return KeepCompiling
end
2023-03-04 23:20:54 -08:00
# vm_call_opt_struct_aref
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-04 23:20:54 -08:00
def jit_call_opt_struct_aref(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:)
if argc != 0
asm.incr_counter(:send_optimized_struct_aref_error)
return CantCompile
end
off = cme.def.body.optimized.index
recv_idx = argc # blockarg is not supported
recv_idx += send_shift
comptime_recv = jit.peek_at_stack(recv_idx)
# This is a .send call and we need to adjust the stack
if flags & C::VM_CALL_OPT_SEND != 0
2023-03-04 23:20:54 -08:00
jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:)
end
# All structs from the same Struct class should have the same
# length. So if our comptime_recv is embedded all runtime
# structs of the same class should be as well, and the same is
# true of the converse.
embedded = C::FL_TEST_RAW(comptime_recv, C::RSTRUCT_EMBED_LEN_MASK)
2023-03-04 23:20:54 -08:00
asm.comment('struct aref')
asm.mov(:rax, ctx.stack_pop(1)) # recv
if embedded
asm.mov(:rax, [:rax, C.RStruct.offsetof(:as, :ary) + (C.VALUE.size * off)])
else
asm.mov(:rax, [:rax, C.RStruct.offsetof(:as, :heap, :ptr)])
asm.mov(:rax, [:rax, C.VALUE.size * off])
end
ret = ctx.stack_push
asm.mov(ret, :rax)
jump_to_next_insn(jit, ctx, asm)
EndBlock
end
2023-03-21 22:55:23 -07:00
# vm_call_opt_send (lazy part)
2023-03-06 23:15:30 -08:00
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-16 22:29:58 -08:00
def jit_call_opt_send_shift_stack(ctx, asm, argc, send_shift:)
# We don't support `send(:send, ...)` for now.
assert_equal(1, send_shift)
asm.comment('shift stack')
(0...argc).reverse_each do |i|
opnd = ctx.stack_opnd(i)
opnd2 = ctx.stack_opnd(i + 1)
asm.mov(:rax, opnd)
asm.mov(opnd2, :rax)
end
ctx.stack_pop(1)
end
# vm_call_symbol
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_call_symbol(jit, ctx, asm, cme, calling, known_recv_class, flags)
flags |= C::VM_CALL_OPT_SEND | (calling.kw_splat ? C::VM_CALL_KW_SPLAT : 0)
2023-02-16 22:29:58 -08:00
comptime_symbol = jit.peek_at_stack(calling.argc)
2023-02-16 22:29:58 -08:00
if comptime_symbol.class != String && !static_symbol?(comptime_symbol)
asm.incr_counter(:send_optimized_send_not_sym_or_str)
return CantCompile
end
mid = C.get_symbol_id(comptime_symbol)
if mid == 0
asm.incr_counter(:send_optimized_send_null_mid)
return CantCompile
end
asm.comment("Guard #{comptime_symbol.inspect} is on stack")
2023-03-01 23:29:32 -08:00
class_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_class_changed)
jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_symbol), ctx.stack_opnd(calling.argc), comptime_symbol, class_changed_exit)
asm.mov(C_ARGS[0], ctx.stack_opnd(calling.argc))
2023-02-16 22:29:58 -08:00
asm.call(C.rb_get_symbol_id)
asm.cmp(C_RET, mid)
2023-03-01 23:29:32 -08:00
id_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_id_changed)
jit_chain_guard(:jne, jit, ctx, asm, id_changed_exit)
2023-02-16 22:29:58 -08:00
2023-02-24 14:48:02 -08:00
# rb_callable_method_entry_with_refinements
calling.flags = flags
cme, _ = jit_search_method(jit, ctx, asm, mid, calling)
2023-02-24 14:48:02 -08:00
if cme == CantCompile
return CantCompile
end
if flags & C::VM_CALL_FCALL != 0
return jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class)
2023-02-16 22:29:58 -08:00
end
raise NotImplementedError # unreachable for now
end
2023-02-03 22:42:13 -08:00
# vm_push_frame
#
# Frame structure:
# | args | locals | cme/cref | block_handler/prev EP | frame type (EP here) | stack bottom (SP here)
2023-02-09 14:38:41 -08:00
#
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, block_handler, iseq: nil, local_size: 0, stack_max: 0, prev_ep: nil)
# Save caller SP and PC before pushing a callee frame for backtrace and side exits
asm.comment('save SP to caller CFP')
recv_idx = argc # blockarg is already popped
recv_idx += (block_handler == :captured) ? 0 : 1 # receiver is not on stack when captured->self is used
# TODO: consider doing_kw_call
if iseq
# Skip setting this to SP register. This cfp->sp will be copied to SP on leave insn.
asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * -recv_idx)) # Pop receiver and arguments to prepare for side exits
asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], :rax)
else
asm.lea(SP, ctx.sp_opnd(C.VALUE.size * -recv_idx))
asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP)
ctx.sp_offset = recv_idx
end
jit_save_pc(jit, asm, comment: 'save PC to caller CFP')
2023-02-06 15:07:58 -08:00
local_size.times do |i|
asm.comment('set local variables') if i == 0
2023-02-08 09:30:47 -08:00
local_index = ctx.sp_offset + i
2023-02-06 15:07:58 -08:00
asm.mov([SP, C.VALUE.size * local_index], Qnil)
2023-02-03 22:42:13 -08:00
end
2023-01-07 21:24:30 -08:00
2023-02-09 21:59:27 -08:00
asm.comment('set up EP with managing data')
ep_offset = ctx.sp_offset + local_size + 2
2023-02-28 23:22:12 -08:00
# ep[-2]: cref_or_me
2023-02-09 21:59:27 -08:00
asm.mov(:rax, cme.to_i)
asm.mov([SP, C.VALUE.size * (ep_offset - 2)], :rax)
# ep[-1]: block handler or prev env ptr (specval)
2023-03-04 22:42:03 -08:00
if prev_ep
asm.mov(:rax, prev_ep.to_i | 1) # tagged prev ep
asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax)
2023-03-12 22:27:43 -07:00
elsif block_handler == :captured
# Set captured->ep, saving captured in :rcx for captured->self
ep_reg = :rcx
jit_get_lep(jit, asm, reg: ep_reg)
asm.mov(:rcx, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler
asm.and(:rcx, ~0x3) # captured
asm.mov(:rax, [:rcx, C.VALUE.size]) # captured->ep
asm.or(:rax, 0x1) # GC_GUARDED_PTR
asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax)
elsif block_handler == C::VM_BLOCK_HANDLER_NONE
asm.mov([SP, C.VALUE.size * (ep_offset - 1)], C::VM_BLOCK_HANDLER_NONE)
2023-03-04 21:13:01 -08:00
elsif block_handler == C.rb_block_param_proxy
# vm_caller_setup_arg_block: block_code == rb_block_param_proxy
jit_get_lep(jit, asm, reg: :rax) # VM_CF_BLOCK_HANDLER: VM_CF_LEP
asm.mov(:rax, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # VM_CF_BLOCK_HANDLER: VM_ENV_BLOCK_HANDLER
asm.mov([CFP, C.rb_control_frame_t.offsetof(:block_code)], :rax) # reg_cfp->block_code = handler
asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) # return handler;
2023-02-28 23:22:12 -08:00
else # assume blockiseq
asm.mov(:rax, block_handler)
asm.mov([CFP, C.rb_control_frame_t.offsetof(:block_code)], :rax)
asm.lea(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # VM_CFP_TO_CAPTURED_BLOCK
asm.or(:rax, 1) # VM_BH_FROM_ISEQ_BLOCK
asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax)
end
# ep[-0]: ENV_FLAGS
2023-02-09 21:59:27 -08:00
asm.mov([SP, C.VALUE.size * (ep_offset - 0)], frame_type)
2023-02-09 22:02:10 -08:00
asm.comment('set up new frame')
2023-02-09 22:06:51 -08:00
cfp_offset = -C.rb_control_frame_t.size # callee CFP
2023-02-10 21:54:35 -08:00
# For ISEQ, JIT code will set it as needed. However, C func needs 0 there for svar frame detection.
if iseq.nil?
asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:pc)], 0)
end
2023-02-03 22:42:13 -08:00
asm.mov(:rax, iseq.to_i)
2023-02-09 22:06:51 -08:00
asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:iseq)], :rax)
2023-03-12 22:27:43 -07:00
if block_handler == :captured
asm.mov(:rax, [:rcx]) # captured->self
else
self_index = ctx.sp_offset - (1 + argc) # blockarg has been popped
asm.mov(:rax, [SP, C.VALUE.size * self_index])
end
2023-02-09 22:06:51 -08:00
asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:self)], :rax)
2023-02-09 16:25:06 -08:00
asm.lea(:rax, [SP, C.VALUE.size * ep_offset])
2023-02-09 22:06:51 -08:00
asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:ep)], :rax)
asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:block_code)], 0)
2023-02-09 16:25:06 -08:00
# Update SP register only for ISEQ calls. SP-relative operations should be done above this.
sp_reg = iseq ? SP : :rax
asm.lea(sp_reg, [SP, C.VALUE.size * (ctx.sp_offset + local_size + 3)])
asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:sp)], sp_reg)
asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:__bp__)], sp_reg) # TODO: get rid of this!!
2023-02-03 22:42:13 -08:00
2023-02-09 21:49:35 -08:00
# cfp->jit_return is used only for ISEQs
if iseq
# Stub cfp->jit_return
return_ctx = ctx.dup
2023-03-12 22:27:43 -07:00
return_ctx.stack_size -= argc + ((block_handler == :captured) ? 0 : 1) # Pop args and receiver. blockarg has been popped
return_ctx.stack_size += 1 # push callee's return value
2023-02-09 21:49:35 -08:00
return_ctx.sp_offset = 1 # SP is in the position after popping a receiver and arguments
2023-03-04 22:13:20 -08:00
return_ctx.chain_depth = 0
2023-02-09 21:49:35 -08:00
branch_stub = BranchStub.new(
iseq: jit.iseq,
shape: Default,
target0: BranchTarget.new(ctx: return_ctx, pc: jit.pc + jit.insn.len * C.VALUE.size),
)
branch_stub.target0.address = Assembler.new.then do |ocb_asm|
@exit_compiler.compile_branch_stub(return_ctx, ocb_asm, branch_stub, true)
@ocb.write(ocb_asm)
end
branch_stub.compile = proc do |branch_asm|
branch_asm.comment('set jit_return to callee CFP')
branch_asm.stub(branch_stub) do
case branch_stub.shape
in Default
branch_asm.mov(:rax, branch_stub.target0.address)
2023-02-09 22:06:51 -08:00
branch_asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:jit_return)], :rax)
2023-02-09 21:49:35 -08:00
end
2023-02-07 11:36:45 -08:00
end
2023-01-07 21:24:30 -08:00
end
2023-02-09 21:49:35 -08:00
branch_stub.compile.call(asm)
2023-01-07 21:24:30 -08:00
end
2023-02-09 22:06:51 -08:00
asm.comment('switch to callee CFP')
2023-02-09 16:25:06 -08:00
# Update CFP register only for ISEQ calls
cfp_reg = iseq ? CFP : :rax
asm.lea(cfp_reg, [CFP, cfp_offset])
asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], cfp_reg)
2023-01-07 21:24:30 -08:00
end
2023-02-03 22:42:13 -08:00
# CALLER_SETUP_ARG: Return CantCompile if not supported
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-03-21 22:55:23 -07:00
def jit_caller_setup_arg(jit, ctx, asm, flags, splat: false)
if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_KW_SPLAT != 0
asm.incr_counter(:send_args_splat_kw_splat)
return CantCompile
elsif flags & C::VM_CALL_ARGS_SPLAT != 0
2023-03-21 22:55:23 -07:00
if splat
# Lazily handle splat in jit_call_cfunc_with_frame
2023-03-21 22:55:23 -07:00
else
# splat is not supported in this path
asm.incr_counter(:send_args_splat)
return CantCompile
end
elsif flags & C::VM_CALL_KW_SPLAT != 0
asm.incr_counter(:send_args_kw_splat)
return CantCompile
elsif flags & C::VM_CALL_KWARG != 0
2023-02-06 15:07:58 -08:00
asm.incr_counter(:send_kwarg)
2023-02-03 22:42:13 -08:00
return CantCompile
end
end
2023-03-21 22:55:23 -07:00
# vm_caller_setup_arg_splat (+ CALLER_SETUP_ARG):
# Pushes arguments from an array to the stack that are passed with a splat (i.e. *args).
# It optimistically compiles to a static size that is the exact number of arguments needed for the function.
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
def jit_caller_setup_arg_splat(jit, ctx, asm, required_args)
side_exit = side_exit(jit, ctx)
asm.comment('push_splat_args')
array_opnd = ctx.stack_opnd(0)
array_reg = :rax
asm.mov(array_reg, array_opnd)
guard_object_is_heap(asm, array_reg, counted_exit(side_exit, :send_args_splat_not_array))
guard_object_is_array(asm, array_reg, :rcx, counted_exit(side_exit, :send_args_splat_not_array))
array_len_opnd = :rcx
jit_array_len(asm, array_reg, array_len_opnd)
asm.comment('Side exit if length is not equal to remaining args')
asm.cmp(array_len_opnd, required_args)
asm.jne(counted_exit(side_exit, :send_args_splat_length_not_equal))
asm.comment('Check last argument is not ruby2keyword hash')
ary_opnd = :rcx
jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg
last_array_value = :rax
asm.mov(last_array_value, [ary_opnd, (required_args - 1) * C.VALUE.size])
ruby2_exit = counted_exit(side_exit, :send_args_splat_ruby2_hash);
guard_object_is_not_ruby2_keyword_hash(asm, last_array_value, :rcx, ruby2_exit) # clobbers :rax
asm.comment('Push arguments from array')
array_opnd = ctx.stack_pop(1)
if required_args > 0
# Load the address of the embedded array
# (struct RArray *)(obj)->as.ary
array_reg = :rax
asm.mov(array_reg, array_opnd)
# Conditionally load the address of the heap array
# (struct RArray *)(obj)->as.heap.ptr
flags_opnd = [array_reg, C.RBasic.offsetof(:flags)]
asm.test(flags_opnd, C::RARRAY_EMBED_FLAG)
heap_ptr_opnd = [array_reg, C.RArray.offsetof(:as, :heap, :ptr)]
# Load the address of the embedded array
# (struct RArray *)(obj)->as.ary
asm.lea(:rcx, [array_reg, C.RArray.offsetof(:as, :ary)])
asm.mov(:rax, heap_ptr_opnd)
asm.cmovnz(:rax, :rcx)
ary_opnd = :rax
(0...required_args).each do |i|
top = ctx.stack_push
asm.mov(:rcx, [ary_opnd, i * C.VALUE.size])
asm.mov(top, :rcx)
end
asm.comment('end push_each')
end
end
2023-02-03 22:42:13 -08:00
# CALLER_REMOVE_EMPTY_KW_SPLAT: Return CantCompile if not supported
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2023-02-03 22:42:13 -08:00
def jit_caller_remove_empty_kw_splat(jit, ctx, asm, flags)
if (flags & C::VM_CALL_KW_SPLAT) > 0
2023-02-03 22:42:13 -08:00
# We don't support removing the last Hash argument
2023-02-07 14:42:58 -08:00
asm.incr_counter(:send_kw_splat)
2023-02-03 22:42:13 -08:00
return CantCompile
end
end
2023-02-15 21:26:04 -08:00
# Generate RARRAY_LEN. For array_opnd, use Opnd::Reg to reduce memory access,
# and use Opnd::Mem to save registers.
def jit_array_len(asm, array_reg, len_reg)
asm.comment('get array length for embedded or heap')
# Pull out the embed flag to check if it's an embedded array.
asm.mov(len_reg, [array_reg, C.RBasic.offsetof(:flags)])
# Get the length of the array
asm.and(len_reg, C::RARRAY_EMBED_LEN_MASK)
asm.sar(len_reg, C::RARRAY_EMBED_LEN_SHIFT)
2023-02-15 21:26:04 -08:00
# Conditionally move the length of the heap array
asm.test([array_reg, C.RBasic.offsetof(:flags)], C::RARRAY_EMBED_FLAG)
2023-02-15 21:26:04 -08:00
# Select the array length value
asm.cmovz(len_reg, [array_reg, C.RArray.offsetof(:as, :heap, :len)])
end
2023-03-21 22:55:23 -07:00
# Generate RARRAY_CONST_PTR_TRANSIENT (part of RARRAY_AREF)
def jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg
2023-03-21 22:55:23 -07:00
asm.comment('get array pointer for embedded or heap')
flags_opnd = [array_reg, C.RBasic.offsetof(:flags)]
asm.test(flags_opnd, C::RARRAY_EMBED_FLAG)
# Load the address of the embedded array
# (struct RArray *)(obj)->as.ary
asm.mov(ary_opnd, [array_reg, C.RArray.offsetof(:as, :heap, :ptr)])
asm.lea(array_reg, [array_reg, C.RArray.offsetof(:as, :ary)]) # clobbers array_reg
2023-03-21 22:55:23 -07:00
asm.cmovnz(ary_opnd, array_reg)
end
2023-02-03 22:42:13 -08:00
def assert_equal(left, right)
2022-12-23 14:17:32 -08:00
if left != right
raise "'#{left.inspect}' was not '#{right.inspect}'"
end
end
2022-12-30 22:16:07 -08:00
2023-01-02 22:53:14 -08:00
def fixnum?(obj)
(C.to_value(obj) & C::RUBY_FIXNUM_FLAG) == C::RUBY_FIXNUM_FLAG
2023-02-16 22:29:58 -08:00
end
2023-03-01 23:23:02 -08:00
def flonum?(obj)
(C.to_value(obj) & C::RUBY_FLONUM_MASK) == C::RUBY_FLONUM_FLAG
2023-03-01 23:23:02 -08:00
end
2023-03-12 22:27:43 -07:00
def symbol?(obj)
static_symbol?(obj) || dynamic_symbol?(obj)
end
2023-02-16 22:29:58 -08:00
def static_symbol?(obj)
(C.to_value(obj) & 0xff) == C::RUBY_SYMBOL_FLAG
2023-01-02 22:53:14 -08:00
end
2023-03-12 22:27:43 -07:00
def dynamic_symbol?(obj)
return false if C::SPECIAL_CONST_P(obj)
2023-03-18 22:57:31 -07:00
C.RB_TYPE_P(obj, C::RUBY_T_SYMBOL)
2023-03-12 22:27:43 -07:00
end
2023-03-02 21:14:26 -08:00
def shape_too_complex?(obj)
C.rb_shape_get_shape_id(obj) == C::OBJ_TOO_COMPLEX_SHAPE_ID
2023-03-02 21:14:26 -08:00
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
# @param asm [RubyVM::RJIT::Assembler]
2022-12-31 13:41:32 -08:00
def defer_compilation(jit, ctx, asm)
# Make a stub to compile the current insn
if ctx.chain_depth != 0
raise "double defer!"
end
ctx.chain_depth += 1
jit_direct_jump(jit.iseq, jit.pc, ctx, asm, comment: 'defer_compilation')
2022-12-31 13:41:32 -08:00
end
def jit_direct_jump(iseq, pc, ctx, asm, comment: 'jit_direct_jump')
2023-02-07 11:51:11 -08:00
branch_stub = BranchStub.new(
iseq:,
shape: Default,
target0: BranchTarget.new(ctx:, pc:),
)
branch_stub.target0.address = Assembler.new.then do |ocb_asm|
@exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true)
@ocb.write(ocb_asm)
end
branch_stub.compile = proc do |branch_asm|
branch_asm.comment(comment)
branch_asm.stub(branch_stub) do
case branch_stub.shape
in Default
branch_asm.jmp(branch_stub.target0.address)
in Next0
# Just write the block without a jump
end
end
end
branch_stub.compile.call(asm)
end
2023-03-06 23:15:30 -08:00
# @param jit [RubyVM::RJIT::JITState]
# @param ctx [RubyVM::RJIT::Context]
2023-01-02 22:53:14 -08:00
def side_exit(jit, ctx)
# We use the latest ctx.sp_offset to generate a side exit to tolerate sp_offset changes by jit_save_sp.
# However, we want to simulate an old stack_size when we take a side exit. We do that by adjusting the
# sp_offset because gen_outlined_exit uses ctx.sp_offset to move SP.
ctx = ctx.with_stack_size(jit.stack_size_for_pc)
jit.side_exit_for_pc[ctx.sp_offset] ||= Assembler.new.then do |asm|
@exit_compiler.compile_side_exit(jit.pc, ctx, asm)
@ocb.write(asm)
2023-01-02 22:53:14 -08:00
end
2022-12-30 22:16:07 -08:00
end
2023-01-07 21:24:30 -08:00
2023-02-07 23:32:13 -08:00
def counted_exit(side_exit, name)
asm = Assembler.new
asm.incr_counter(name)
asm.jmp(side_exit)
@ocb.write(asm)
end
2023-01-07 21:24:30 -08:00
def def_iseq_ptr(cme_def)
C.rb_iseq_check(cme_def.body.iseq.iseqptr)
end
2023-02-08 01:48:32 -08:00
def to_value(obj)
2023-02-21 00:19:37 -08:00
GC_REFS << obj
2023-02-08 01:48:32 -08:00
C.to_value(obj)
end
2023-03-09 21:55:14 -08:00
def full_cfunc_return
@full_cfunc_return ||= Assembler.new.then do |asm|
@exit_compiler.compile_full_cfunc_return(asm)
@ocb.write(asm)
end
end
2023-03-26 17:41:05 -07:00
def c_method_tracing_currently_enabled?
C.rb_rjit_global_events & (C::RUBY_EVENT_C_CALL | C::RUBY_EVENT_C_RETURN) != 0
end
2023-03-26 19:20:03 -07:00
# Return a builtin function if a given iseq consists of only that builtin function
def builtin_function(iseq)
opt_invokebuiltin_delegate_leave = INSNS.values.find { |i| i.name == :opt_invokebuiltin_delegate_leave }
leave = INSNS.values.find { |i| i.name == :leave }
if iseq.body.iseq_size == opt_invokebuiltin_delegate_leave.len + leave.len &&
C.rb_vm_insn_decode(iseq.body.iseq_encoded[0]) == opt_invokebuiltin_delegate_leave.bin &&
C.rb_vm_insn_decode(iseq.body.iseq_encoded[opt_invokebuiltin_delegate_leave.len]) == leave.bin
C.rb_builtin_function.new(iseq.body.iseq_encoded[1])
end
end
def build_calling(ci:, block_handler:)
CallingInfo.new(
argc: C.vm_ci_argc(ci),
flags: C.vm_ci_flag(ci),
kwarg: C.vm_ci_kwarg(ci),
send_shift: 0,
block_handler:,
)
end
2022-12-17 13:39:35 -08:00
end
end