10#include "internal/sanitizers.h"
11#include "internal/string.h"
12#include "internal/hash.h"
13#include "internal/variable.h"
14#include "internal/compile.h"
15#include "internal/class.h"
16#include "internal/fixnum.h"
19#include "vm_callinfo.h"
22#include "insns_info.inc"
25#include "vm_insnhelper.h"
27#include "probes_helper.h"
30#include "internal/cont.h"
42STATIC_ASSERT(64b_size_t, SIZE_MAX == UINT64_MAX);
45STATIC_ASSERT(size_t_no_padding_bits,
sizeof(
size_t) ==
sizeof(uint64_t));
49STATIC_ASSERT(pointer_tagging_scheme,
USE_FLONUM);
67rb_yjit_mark_writable(
void *mem_block, uint32_t mem_size)
69 return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0;
73rb_yjit_mark_executable(
void *mem_block, uint32_t mem_size)
80 if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
81 rb_bug(
"Couldn't make JIT page (%p, %lu bytes) executable, errno: %s\n",
82 mem_block, (
unsigned long)mem_size, strerror(errno));
88rb_yjit_mark_unused(
void *mem_block, uint32_t mem_size)
93 madvise(mem_block, mem_size, MADV_DONTNEED);
97 return mprotect(mem_block, mem_size, PROT_NONE) == 0;
102rb_yjit_icache_invalidate(
void *start,
void *end)
108 __builtin___clear_cache(start, end);
109#elif defined(__aarch64__)
110#error No instruction cache clear available with this compiler on Aarch64!
114# define PTR2NUM(x) (rb_int2inum((intptr_t)(void *)(x)))
121 VALUE frame_id = PTR2NUM(frame);
123 if (
RTEST(rb_hash_aref(hash, frame_id))) {
127 VALUE frame_info = rb_hash_new();
151 rb_hash_aset(hash, frame_id, frame_info);
161rb_yjit_exit_locations_dict(
VALUE *yjit_raw_samples,
int *yjit_line_samples,
int samples_len)
163 VALUE result = rb_hash_new();
164 VALUE raw_samples = rb_ary_new_capa(samples_len);
165 VALUE line_samples = rb_ary_new_capa(samples_len);
166 VALUE frames = rb_hash_new();
171 while (idx < samples_len) {
172 int num = (int)yjit_raw_samples[idx];
173 int line_num = (int)yjit_line_samples[idx];
176 rb_ary_push(raw_samples,
SIZET2NUM(num));
177 rb_ary_push(line_samples,
INT2NUM(line_num));
182 for (
int o = 0; o < num; o++) {
183 rb_yjit_add_frame(frames, yjit_raw_samples[idx]);
184 rb_ary_push(raw_samples,
SIZET2NUM(yjit_raw_samples[idx]));
185 rb_ary_push(line_samples,
INT2NUM(yjit_line_samples[idx]));
189 rb_ary_push(raw_samples,
SIZET2NUM(yjit_raw_samples[idx]));
190 rb_ary_push(line_samples,
INT2NUM(yjit_line_samples[idx]));
193 rb_ary_push(raw_samples,
SIZET2NUM(yjit_raw_samples[idx]));
194 rb_ary_push(line_samples,
INT2NUM(yjit_line_samples[idx]));
208rb_yjit_get_page_size(
void)
210#if defined(_SC_PAGESIZE)
211 long page_size = sysconf(_SC_PAGESIZE);
212 if (page_size <= 0)
rb_bug(
"yjit: failed to get page size");
217 if (page_size > 0x40000000l)
rb_bug(
"yjit page size too large");
219 return (uint32_t)page_size;
221#error "YJIT supports POSIX only for now"
225#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
228align_ptr(uint8_t *ptr, uint32_t multiple)
231 uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
238 uint32_t pad = multiple - rem;
247rb_yjit_reserve_addr_space(uint32_t mem_size)
253 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
254 uint32_t
const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
255 uint8_t *
const cfunc_sample_addr = (
void *)&rb_yjit_reserve_addr_space;
256 uint8_t *
const probe_region_end = cfunc_sample_addr + INT32_MAX;
258 uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
267 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
273 if (mem_block != MAP_FAILED) {
278 req_addr += 4 * 1024 * 1024;
279 }
while (req_addr < probe_region_end);
285 (
void *)rb_yjit_reserve_addr_space,
288 MAP_PRIVATE | MAP_ANONYMOUS,
295 if (mem_block == MAP_FAILED) {
301 MAP_PRIVATE | MAP_ANONYMOUS,
308 if (mem_block == MAP_FAILED) {
309 perror(
"ruby: yjit: mmap:");
310 if(errno == ENOMEM) {
329 if (rb_multi_ractor_p()) {
330 tracing_events = ruby_vm_event_enabled_global_flags;
336 tracing_events = rb_ec_ractor_hooks(ec)->events;
360 EXEC_EVENT_HOOK(ec,
RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, return_value);
364 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
368 ec->cfp->sp[0] = return_value;
373rb_iseq_encoded_size(
const rb_iseq_t *iseq)
375 return iseq->body->iseq_size;
380rb_iseq_get_yjit_payload(
const rb_iseq_t *iseq)
384 return iseq->body->yjit_payload;
393rb_iseq_set_yjit_payload(
const rb_iseq_t *iseq,
void *payload)
398 iseq->body->yjit_payload = payload;
402rb_iseq_reset_jit_func(
const rb_iseq_t *iseq)
405 iseq->body->jit_func = NULL;
408 iseq->body->total_calls = 0;
413rb_iseq_pc_at_idx(
const rb_iseq_t *iseq, uint32_t insn_idx)
417 VALUE *encoded = iseq->body->iseq_encoded;
418 VALUE *pc = &encoded[insn_idx];
429 const VALUE at_pc = *pc;
430 return rb_vm_insn_addr2opcode((
const void *)at_pc);
435rb_str_bytesize(
VALUE str)
441rb_RSTRING_LEN(
VALUE str)
447rb_RSTRING_PTR(
VALUE str)
453rb_yjit_get_proc_ptr(
VALUE procv)
456 GetProcPtr(procv, proc);
463typedef struct rb_iseq_param_keyword rb_seq_param_keyword_struct;
466rb_insn_name(
VALUE insn)
468 return insn_name(insn);
473rb_insn_len(
VALUE insn)
475 return insn_len(insn);
481 return vm_ci_argc(ci);
487 return vm_ci_mid(ci);
493 return vm_ci_flag(ci);
499 return vm_ci_kwarg(ci);
505 return cikw->keyword_len;
511 return cikw->keywords[idx];
514rb_method_visibility_t
517 return METHOD_ENTRY_VISI(me);
523 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
524 return VM_METHOD_TYPE_UNDEF;
526 return cme->def->type;
533 return cme->def->body.attr.id;
536ID rb_get_symbol_id(
VALUE namep);
538enum method_optimized_type
541 return cme->def->body.optimized.type;
547 return cme->def->body.optimized.index;
553 return UNALIGNED_MEMBER_PTR(cme->def, body.cfunc);
559 return def->method_serial;
565 return def->original_id;
577 return (
void*)mct->func;
583 return def_iseq_ptr(def);
590 return def->body.bmethod.proc;
594rb_get_iseq_body_local_iseq(
const rb_iseq_t *iseq)
596 return iseq->body->local_iseq;
600rb_get_iseq_body_parent_iseq(
const rb_iseq_t *iseq)
602 return iseq->body->parent_iseq;
606rb_get_iseq_body_local_table_size(
const rb_iseq_t *iseq)
608 return iseq->body->local_table_size;
612rb_get_iseq_body_iseq_encoded(
const rb_iseq_t *iseq)
614 return iseq->body->iseq_encoded;
618rb_get_iseq_body_builtin_inline_p(
const rb_iseq_t *iseq)
620 return iseq->body->builtin_inline_p;
624rb_get_iseq_body_stack_max(
const rb_iseq_t *iseq)
626 return iseq->body->stack_max;
630rb_get_iseq_flags_has_lead(
const rb_iseq_t *iseq)
632 return iseq->body->
param.flags.has_lead;
636rb_get_iseq_flags_has_opt(
const rb_iseq_t *iseq)
638 return iseq->body->
param.flags.has_opt;
642rb_get_iseq_flags_has_kw(
const rb_iseq_t *iseq)
644 return iseq->body->
param.flags.has_kw;
648rb_get_iseq_flags_has_post(
const rb_iseq_t *iseq)
650 return iseq->body->
param.flags.has_post;
654rb_get_iseq_flags_has_kwrest(
const rb_iseq_t *iseq)
656 return iseq->body->
param.flags.has_kwrest;
660rb_get_iseq_flags_has_rest(
const rb_iseq_t *iseq)
662 return iseq->body->
param.flags.has_rest;
666rb_get_iseq_flags_ruby2_keywords(
const rb_iseq_t *iseq)
668 return iseq->body->
param.flags.ruby2_keywords;
672rb_get_iseq_flags_has_block(
const rb_iseq_t *iseq)
674 return iseq->body->
param.flags.has_block;
678rb_get_iseq_flags_ambiguous_param0(
const rb_iseq_t *iseq)
680 return iseq->body->
param.flags.ambiguous_param0;
684rb_get_iseq_flags_accepts_no_kwarg(
const rb_iseq_t *iseq)
686 return iseq->body->
param.flags.accepts_no_kwarg;
689const rb_seq_param_keyword_struct *
690rb_get_iseq_body_param_keyword(
const rb_iseq_t *iseq)
692 return iseq->body->
param.keyword;
696rb_get_iseq_body_param_size(
const rb_iseq_t *iseq)
698 return iseq->body->
param.size;
702rb_get_iseq_body_param_lead_num(
const rb_iseq_t *iseq)
704 return iseq->body->
param.lead_num;
708rb_get_iseq_body_param_opt_num(
const rb_iseq_t *iseq)
710 return iseq->body->
param.opt_num;
714rb_get_iseq_body_param_opt_table(
const rb_iseq_t *iseq)
716 return iseq->body->
param.opt_table;
723 GetProcPtr(recv, proc);
724 return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
730rb_leaf_invokebuiltin_iseq_p(
const rb_iseq_t *iseq)
732 unsigned int invokebuiltin_len = insn_len(BIN(opt_invokebuiltin_delegate_leave));
733 unsigned int leave_len = insn_len(BIN(leave));
735 return (iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
736 rb_vm_insn_addr2opcode((
void *)iseq->body->iseq_encoded[0]) == BIN(opt_invokebuiltin_delegate_leave) &&
737 rb_vm_insn_addr2opcode((
void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave) &&
738 iseq->body->builtin_inline_p
744rb_leaf_builtin_function(
const rb_iseq_t *iseq)
746 if (!rb_leaf_invokebuiltin_iseq_p(iseq))
752rb_yjit_str_simple_append(
VALUE str1,
VALUE str2)
766 return (
VALUE*)cfp->pc;
803 return (
VALUE*)cfp->ep;
811 for (i = 0; i < lv; i++) {
812 ep = VM_ENV_PREV_EP(ep);
818rb_yarv_class_of(
VALUE obj)
825rb_yarv_str_eql_internal(
VALUE str1,
VALUE str2)
828 return rb_str_eql_internal(str1, str2);
833rb_yarv_ary_entry_internal(
VALUE ary,
long offset)
835 return rb_ary_entry_internal(ary, offset);
841 return rb_fix_mod_fix(recv, obj);
846rb_yjit_dump_iseq_loc(
const rb_iseq_t *iseq, uint32_t insn_idx)
850 VALUE path = rb_iseq_path(iseq);
852 fprintf(stderr,
"%s %.*s:%u\n", __func__, (
int)len, ptr, rb_iseq_line_no(iseq, insn_idx));
877rb_RSTRUCT_LEN(
VALUE st)
879 return RSTRUCT_LEN(st);
888 RSTRUCT_SET(st, k, v);
898rb_BASIC_OP_UNREDEFINED_P(
enum ruby_basic_operators bop, uint32_t klass)
900 return BASIC_OP_UNREDEFINED_P(bop, klass);
904rb_RCLASS_ORIGIN(
VALUE c)
906 return RCLASS_ORIGIN(c);
911rb_ENCODING_GET(
VALUE obj)
917rb_yjit_multi_ractor_p(
void)
919 return rb_multi_ractor_p();
924rb_assert_iseq_handle(
VALUE handle)
931rb_IMEMO_TYPE_P(
VALUE imemo,
enum imemo_type imemo_type)
933 return IMEMO_TYPE_P(imemo, imemo_type);
937rb_assert_cme_handle(
VALUE handle)
945 rb_iseq_callback callback;
951for_each_iseq_i(
void *vstart,
void *vend,
size_t stride,
void *data)
955 for (; v != (
VALUE)vend; v += stride) {
956 void *ptr = asan_poisoned_object_p(v);
957 asan_unpoison_object(v,
false);
959 if (rb_obj_is_iseq(v)) {
961 callback_data->callback(iseq, callback_data->data);
964 asan_poison_object_if(ptr, v);
972rb_yjit_for_each_iseq(rb_iseq_callback callback,
void *data)
975 rb_objspace_each_objects(for_each_iseq_i, (
void *)&callback_data);
981rb_yjit_obj_written(
VALUE old,
VALUE young,
const char *file,
int line)
983 rb_obj_written(old,
Qundef, young, file, line);
991rb_yjit_vm_lock_then_barrier(
unsigned int *recursive_lock_level,
const char *file,
int line)
993 rb_vm_lock_enter(recursive_lock_level, file, line);
1000rb_yjit_vm_unlock(
unsigned int *recursive_lock_level,
const char *file,
int line)
1002 rb_vm_lock_leave(recursive_lock_level, file, line);
1011 bool success =
true;
1017 uint8_t *code_ptr = rb_yjit_iseq_gen_entry_point(iseq, ec);
1020 iseq->body->jit_func = (yjit_func_t)code_ptr;
1023 iseq->body->jit_func = 0;
1037yjit_root_free(
void *ptr)
1043yjit_root_memsize(
const void *ptr)
1051yjit_root_update_references(
void *ptr)
1056void rb_yjit_root_mark(
void *ptr);
1062 {rb_yjit_root_mark, yjit_root_free, yjit_root_memsize, yjit_root_update_references},
1063 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1068rb_yjit_invalidate_all_method_lookup_assumptions(
void)
1079 return ULONG2NUM((
unsigned long)GET_VM()->next_shape_id);
1094#include "yjit.rbinc"
1101 void rb_yjit_init_rust(
void);
1102 rb_yjit_init_rust();
1107 rb_gc_register_mark_object(yjit_root);
#define RUBY_ASSERT(expr)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
VALUE rb_profile_frame_full_label(VALUE frame)
Identical to rb_profile_frame_label(), except it returns a qualified result.
VALUE rb_profile_frame_absolute_path(VALUE frame)
Identical to rb_profile_frame_path(), except it tries to expand the returning path.
VALUE rb_profile_frame_path(VALUE frame)
Queries the path of the passed backtrace.
VALUE rb_profile_frame_first_lineno(VALUE frame)
Queries the first line of the method of the passed frame pointer.
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
uint32_t rb_event_flag_t
Represents event(s).
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
#define ID2SYM
Old name of RB_ID2SYM.
#define ULONG2NUM
Old name of RB_ULONG2NUM.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
#define LONG2NUM
Old name of RB_LONG2NUM.
#define INT2NUM
Old name of RB_INT2NUM.
#define NIL_P
Old name of RB_NIL_P.
void rb_bug(const char *fmt,...)
Interpreter panic switch.
static VALUE rb_class_of(VALUE obj)
Object to class mapping function.
static int RB_ENCODING_GET(VALUE obj)
Just another name of rb_enc_get_index.
VALUE rb_str_cat(VALUE dst, const char *src, long srclen)
Destructively appends the passed contents to the string.
ID rb_intern(const char *name)
Finds or creates a symbol of the given name.
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
#define TypedData_Make_Struct(klass, type, data_type, sval)
Identical to TypedData_Wrap_Struct, except it allocates a new data region internally instead of takin...
#define RTEST
This is an old name of RB_TEST.
This is the struct that holds necessary info for a struct.
struct rb_iseq_constant_body::@132 param
parameter information
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
ruby_value_type
C-level type of an object.