Ruby 3.3.5p100 (2024-09-03 revision ef084cc8f4958c1b6e4ead99136631bef6d8ddba)
yjit.c
1// This part of YJIT helps interfacing with the rest of CRuby and with the OS.
2// Sometimes our FFI binding generation tool gives undesirable outputs when it
3// sees C features that Rust doesn't support well. We mitigate that by binding
4// functions which have simple parameter types. The boilerplate C functions for
5// that purpose are in this file.
6// Similarly, we wrap OS facilities we need in simple functions to help with
7// FFI and to avoid the need to use external crates.io Rust libraries.
8
9#include "internal.h"
10#include "internal/sanitizers.h"
11#include "internal/string.h"
12#include "internal/hash.h"
13#include "internal/variable.h"
14#include "internal/compile.h"
15#include "internal/class.h"
16#include "internal/fixnum.h"
17#include "internal/numeric.h"
18#include "internal/gc.h"
19#include "vm_core.h"
20#include "vm_callinfo.h"
21#include "builtin.h"
22#include "insns.inc"
23#include "insns_info.inc"
24#include "vm_sync.h"
25#include "yjit.h"
26#include "vm_insnhelper.h"
27#include "probes.h"
28#include "probes_helper.h"
29#include "iseq.h"
30#include "ruby/debug.h"
31#include "internal/cont.h"
32
33// For mmapp(), sysconf()
34#ifndef _WIN32
35#include <unistd.h>
36#include <sys/mman.h>
37#endif
38
39#include <errno.h>
40
41// Field offsets for the RObject struct
42enum robject_offsets {
43 ROBJECT_OFFSET_AS_HEAP_IVPTR = offsetof(struct RObject, as.heap.ivptr),
44 ROBJECT_OFFSET_AS_HEAP_IV_INDEX_TBL = offsetof(struct RObject, as.heap.iv_index_tbl),
45 ROBJECT_OFFSET_AS_ARY = offsetof(struct RObject, as.ary),
46};
47
48// Field offsets for the RString struct
49enum rstring_offsets {
50 RUBY_OFFSET_RSTRING_LEN = offsetof(struct RString, len)
51};
52
53// We need size_t to have a known size to simplify code generation and FFI.
54// TODO(alan): check this in configure.ac to fail fast on 32 bit platforms.
55STATIC_ASSERT(64b_size_t, SIZE_MAX == UINT64_MAX);
56// I don't know any C implementation that has uint64_t and puts padding bits
57// into size_t but the standard seems to allow it.
58STATIC_ASSERT(size_t_no_padding_bits, sizeof(size_t) == sizeof(uint64_t));
59
60// This build config impacts the pointer tagging scheme and we only want to
61// support one scheme for simplicity.
62STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM);
63
64// NOTE: We can trust that uint8_t has no "padding bits" since the C spec
65// guarantees it. Wording about padding bits is more explicit in C11 compared
66// to C99. See C11 7.20.1.1p2. All this is to say we have _some_ standards backing to
67// use a Rust `*mut u8` to represent a C `uint8_t *`.
68//
69// If we don't want to trust that we can interpreter the C standard correctly, we
70// could outsource that work to the Rust standard library by sticking to fundamental
71// types in C such as int, long, etc. and use `std::os::raw::c_long` and friends on
72// the Rust side.
73//
74// What's up with the long prefix? Even though we build with `-fvisibility=hidden`
75// we are sometimes a static library where the option doesn't prevent name collision.
76// The "_yjit_" part is for trying to be informative. We might want different
77// suffixes for symbols meant for Rust and symbols meant for broader CRuby.
78
79bool
80rb_yjit_mark_writable(void *mem_block, uint32_t mem_size)
81{
82 return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0;
83}
84
85void
86rb_yjit_mark_executable(void *mem_block, uint32_t mem_size)
87{
88 // Do not call mprotect when mem_size is zero. Some platforms may return
89 // an error for it. https://github.com/Shopify/ruby/issues/450
90 if (mem_size == 0) {
91 return;
92 }
93 if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
94 rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s",
95 mem_block, (unsigned long)mem_size, strerror(errno));
96 }
97}
98
99// Free the specified memory block.
100bool
101rb_yjit_mark_unused(void *mem_block, uint32_t mem_size)
102{
103 // On Linux, you need to use madvise MADV_DONTNEED to free memory.
104 // We might not need to call this on macOS, but it's not really documented.
105 // We generally prefer to do the same thing on both to ease testing too.
106 madvise(mem_block, mem_size, MADV_DONTNEED);
107
108 // On macOS, mprotect PROT_NONE seems to reduce RSS.
109 // We also call this on Linux to avoid executing unused pages.
110 return mprotect(mem_block, mem_size, PROT_NONE) == 0;
111}
112
113long
114rb_yjit_array_len(VALUE a)
115{
116 return rb_array_len(a);
117}
118
119// `start` is inclusive and `end` is exclusive.
120void
121rb_yjit_icache_invalidate(void *start, void *end)
122{
123 // Clear/invalidate the instruction cache. Compiles to nothing on x86_64
124 // but required on ARM before running freshly written code.
125 // On Darwin it's the same as calling sys_icache_invalidate().
126#ifdef __GNUC__
127 __builtin___clear_cache(start, end);
128#elif defined(__aarch64__)
129#error No instruction cache clear available with this compiler on Aarch64!
130#endif
131}
132
133# define PTR2NUM(x) (rb_int2inum((intptr_t)(void *)(x)))
134
135// For a given raw_sample (frame), set the hash with the caller's
136// name, file, and line number. Return the hash with collected frame_info.
137static void
138rb_yjit_add_frame(VALUE hash, VALUE frame)
139{
140 VALUE frame_id = PTR2NUM(frame);
141
142 if (RTEST(rb_hash_aref(hash, frame_id))) {
143 return;
144 }
145 else {
146 VALUE frame_info = rb_hash_new();
147 // Full label for the frame
149 // Absolute path of the frame from rb_iseq_realpath
151 // Line number of the frame
153
154 // If absolute path isn't available use the rb_iseq_path
155 if (NIL_P(file)) {
156 file = rb_profile_frame_path(frame);
157 }
158
159 rb_hash_aset(frame_info, ID2SYM(rb_intern("name")), name);
160 rb_hash_aset(frame_info, ID2SYM(rb_intern("file")), file);
161 rb_hash_aset(frame_info, ID2SYM(rb_intern("samples")), INT2NUM(0));
162 rb_hash_aset(frame_info, ID2SYM(rb_intern("total_samples")), INT2NUM(0));
163 rb_hash_aset(frame_info, ID2SYM(rb_intern("edges")), rb_hash_new());
164 rb_hash_aset(frame_info, ID2SYM(rb_intern("lines")), rb_hash_new());
165
166 if (line != INT2FIX(0)) {
167 rb_hash_aset(frame_info, ID2SYM(rb_intern("line")), line);
168 }
169
170 rb_hash_aset(hash, frame_id, frame_info);
171 }
172}
173
174// Parses the YjitExitLocations raw_samples and line_samples collected by
175// rb_yjit_record_exit_stack and turns them into 3 hashes (raw, lines, and frames) to
176// be used by RubyVM::YJIT.exit_locations. yjit_raw_samples represents the raw frames information
177// (without name, file, and line), and yjit_line_samples represents the line information
178// of the iseq caller.
179VALUE
180rb_yjit_exit_locations_dict(VALUE *yjit_raw_samples, int *yjit_line_samples, int samples_len)
181{
182 VALUE result = rb_hash_new();
183 VALUE raw_samples = rb_ary_new_capa(samples_len);
184 VALUE line_samples = rb_ary_new_capa(samples_len);
185 VALUE frames = rb_hash_new();
186 int idx = 0;
187
188 // While the index is less than samples_len, parse yjit_raw_samples and
189 // yjit_line_samples, then add casted values to raw_samples and line_samples array.
190 while (idx < samples_len) {
191 int num = (int)yjit_raw_samples[idx];
192 int line_num = (int)yjit_line_samples[idx];
193 idx++;
194
195 // + 1 as we append an additional sample for the insn
196 rb_ary_push(raw_samples, SIZET2NUM(num + 1));
197 rb_ary_push(line_samples, INT2NUM(line_num + 1));
198
199 // Loop through the length of samples_len and add data to the
200 // frames hash. Also push the current value onto the raw_samples
201 // and line_samples array respectively.
202 for (int o = 0; o < num; o++) {
203 rb_yjit_add_frame(frames, yjit_raw_samples[idx]);
204 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
205 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
206 idx++;
207 }
208
209 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
210 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
211 idx++;
212
213 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
214 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
215 idx++;
216 }
217
218 // Set add the raw_samples, line_samples, and frames to the results
219 // hash.
220 rb_hash_aset(result, ID2SYM(rb_intern("raw")), raw_samples);
221 rb_hash_aset(result, ID2SYM(rb_intern("lines")), line_samples);
222 rb_hash_aset(result, ID2SYM(rb_intern("frames")), frames);
223
224 return result;
225}
226
227uint32_t
228rb_yjit_get_page_size(void)
229{
230#if defined(_SC_PAGESIZE)
231 long page_size = sysconf(_SC_PAGESIZE);
232 if (page_size <= 0) rb_bug("yjit: failed to get page size");
233
234 // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected.
235 // Though our design sort of assume we have fine grained control over memory protection
236 // which require small page sizes.
237 if (page_size > 0x40000000l) rb_bug("yjit page size too large");
238
239 return (uint32_t)page_size;
240#else
241#error "YJIT supports POSIX only for now"
242#endif
243}
244
245#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
246// Align the current write position to a multiple of bytes
247static uint8_t *
248align_ptr(uint8_t *ptr, uint32_t multiple)
249{
250 // Compute the pointer modulo the given alignment boundary
251 uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
252
253 // If the pointer is already aligned, stop
254 if (rem == 0)
255 return ptr;
256
257 // Pad the pointer by the necessary amount to align it
258 uint32_t pad = multiple - rem;
259
260 return ptr + pad;
261}
262#endif
263
264// Address space reservation. Memory pages are mapped on an as needed basis.
265// See the Rust mm module for details.
266uint8_t *
267rb_yjit_reserve_addr_space(uint32_t mem_size)
268{
269#ifndef _WIN32
270 uint8_t *mem_block;
271
272 // On Linux
273 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
274 uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
275 uint8_t *const cfunc_sample_addr = (void *)&rb_yjit_reserve_addr_space;
276 uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX;
277 // Align the requested address to page size
278 uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
279
280 // Probe for addresses close to this function using MAP_FIXED_NOREPLACE
281 // to improve odds of being in range for 32-bit relative call instructions.
282 do {
283 mem_block = mmap(
284 req_addr,
285 mem_size,
286 PROT_NONE,
287 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
288 -1,
289 0
290 );
291
292 // If we succeeded, stop
293 if (mem_block != MAP_FAILED) {
294 break;
295 }
296
297 // +4MB
298 req_addr += 4 * 1024 * 1024;
299 } while (req_addr < probe_region_end);
300
301 // On MacOS and other platforms
302 #else
303 // Try to map a chunk of memory as executable
304 mem_block = mmap(
305 (void *)rb_yjit_reserve_addr_space,
306 mem_size,
307 PROT_NONE,
308 MAP_PRIVATE | MAP_ANONYMOUS,
309 -1,
310 0
311 );
312 #endif
313
314 // Fallback
315 if (mem_block == MAP_FAILED) {
316 // Try again without the address hint (e.g., valgrind)
317 mem_block = mmap(
318 NULL,
319 mem_size,
320 PROT_NONE,
321 MAP_PRIVATE | MAP_ANONYMOUS,
322 -1,
323 0
324 );
325 }
326
327 // Check that the memory mapping was successful
328 if (mem_block == MAP_FAILED) {
329 perror("ruby: yjit: mmap:");
330 if(errno == ENOMEM) {
331 // No crash report if it's only insufficient memory
332 exit(EXIT_FAILURE);
333 }
334 rb_bug("mmap failed");
335 }
336
337 return mem_block;
338#else
339 // Windows not supported for now
340 return NULL;
341#endif
342}
343
344// Is anyone listening for :c_call and :c_return event currently?
345bool
346rb_c_method_tracing_currently_enabled(const rb_execution_context_t *ec)
347{
348 rb_event_flag_t tracing_events;
349 if (rb_multi_ractor_p()) {
350 tracing_events = ruby_vm_event_enabled_global_flags;
351 }
352 else {
353 // At the time of writing, events are never removed from
354 // ruby_vm_event_enabled_global_flags so always checking using it would
355 // mean we don't compile even after tracing is disabled.
356 tracing_events = rb_ec_ractor_hooks(ec)->events;
357 }
358
359 return tracing_events & (RUBY_EVENT_C_CALL | RUBY_EVENT_C_RETURN);
360}
361
362// The code we generate in gen_send_cfunc() doesn't fire the c_return TracePoint event
363// like the interpreter. When tracing for c_return is enabled, we patch the code after
364// the C method return to call into this to fire the event.
365void
366rb_full_cfunc_return(rb_execution_context_t *ec, VALUE return_value)
367{
368 rb_control_frame_t *cfp = ec->cfp;
369 RUBY_ASSERT_ALWAYS(cfp == GET_EC()->cfp);
370 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
371
372 RUBY_ASSERT_ALWAYS(RUBYVM_CFUNC_FRAME_P(cfp));
373 RUBY_ASSERT_ALWAYS(me->def->type == VM_METHOD_TYPE_CFUNC);
374
375 // CHECK_CFP_CONSISTENCY("full_cfunc_return"); TODO revive this
376
377 // Pop the C func's frame and fire the c_return TracePoint event
378 // Note that this is the same order as vm_call_cfunc_with_frame().
379 rb_vm_pop_frame(ec);
380 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, return_value);
381 // Note, this deviates from the interpreter in that users need to enable
382 // a c_return TracePoint for this DTrace hook to work. A reasonable change
383 // since the Ruby return event works this way as well.
384 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
385
386 // Push return value into the caller's stack. We know that it's a frame that
387 // uses cfp->sp because we are patching a call done with gen_send_cfunc().
388 ec->cfp->sp[0] = return_value;
389 ec->cfp->sp++;
390}
391
392unsigned int
393rb_iseq_encoded_size(const rb_iseq_t *iseq)
394{
395 return iseq->body->iseq_size;
396}
397
398// TODO(alan): consider using an opaque pointer for the payload rather than a void pointer
399void *
400rb_iseq_get_yjit_payload(const rb_iseq_t *iseq)
401{
402 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
403 if (iseq->body) {
404 return iseq->body->yjit_payload;
405 }
406 else {
407 // Body is NULL when constructing the iseq.
408 return NULL;
409 }
410}
411
412void
413rb_iseq_set_yjit_payload(const rb_iseq_t *iseq, void *payload)
414{
415 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
416 RUBY_ASSERT_ALWAYS(iseq->body);
417 RUBY_ASSERT_ALWAYS(NULL == iseq->body->yjit_payload);
418 iseq->body->yjit_payload = payload;
419}
420
421void
422rb_iseq_reset_jit_func(const rb_iseq_t *iseq)
423{
424 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
425 iseq->body->jit_entry = NULL;
426 iseq->body->jit_exception = NULL;
427 // Enable re-compiling this ISEQ. Event when it's invalidated for TracePoint,
428 // we'd like to re-compile ISEQs that haven't been converted to trace_* insns.
429 iseq->body->jit_entry_calls = 0;
430 iseq->body->jit_exception_calls = 0;
431}
432
433// Get the PC for a given index in an iseq
434VALUE *
435rb_iseq_pc_at_idx(const rb_iseq_t *iseq, uint32_t insn_idx)
436{
437 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
438 RUBY_ASSERT_ALWAYS(insn_idx < iseq->body->iseq_size);
439 VALUE *encoded = iseq->body->iseq_encoded;
440 VALUE *pc = &encoded[insn_idx];
441 return pc;
442}
443
444// Get the opcode given a program counter. Can return trace opcode variants.
445int
446rb_iseq_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc)
447{
448 // YJIT should only use iseqs after AST to bytecode compilation
449 RUBY_ASSERT_ALWAYS(FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED));
450
451 const VALUE at_pc = *pc;
452 return rb_vm_insn_addr2opcode((const void *)at_pc);
453}
454
455unsigned long
456rb_RSTRING_LEN(VALUE str)
457{
458 return RSTRING_LEN(str);
459}
460
461char *
462rb_RSTRING_PTR(VALUE str)
463{
464 return RSTRING_PTR(str);
465}
466
467rb_proc_t *
468rb_yjit_get_proc_ptr(VALUE procv)
469{
470 rb_proc_t *proc;
471 GetProcPtr(procv, proc);
472 return proc;
473}
474
475// This is defined only as a named struct inside rb_iseq_constant_body.
476// By giving it a separate typedef, we make it nameable by rust-bindgen.
477// Bindgen's temp/anon name isn't guaranteed stable.
478typedef struct rb_iseq_param_keyword rb_seq_param_keyword_struct;
479
480const char *
481rb_insn_name(VALUE insn)
482{
483 return insn_name(insn);
484}
485
486unsigned int
487rb_vm_ci_argc(const struct rb_callinfo *ci)
488{
489 return vm_ci_argc(ci);
490}
491
492ID
493rb_vm_ci_mid(const struct rb_callinfo *ci)
494{
495 return vm_ci_mid(ci);
496}
497
498unsigned int
499rb_vm_ci_flag(const struct rb_callinfo *ci)
500{
501 return vm_ci_flag(ci);
502}
503
504const struct rb_callinfo_kwarg *
505rb_vm_ci_kwarg(const struct rb_callinfo *ci)
506{
507 return vm_ci_kwarg(ci);
508}
509
510int
511rb_get_cikw_keyword_len(const struct rb_callinfo_kwarg *cikw)
512{
513 return cikw->keyword_len;
514}
515
516VALUE
517rb_get_cikw_keywords_idx(const struct rb_callinfo_kwarg *cikw, int idx)
518{
519 return cikw->keywords[idx];
520}
521
522rb_method_visibility_t
523rb_METHOD_ENTRY_VISI(const rb_callable_method_entry_t *me)
524{
525 return METHOD_ENTRY_VISI(me);
526}
527
528rb_method_type_t
529rb_get_cme_def_type(const rb_callable_method_entry_t *cme)
530{
531 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
532 return VM_METHOD_TYPE_UNDEF;
533 }
534 else {
535 return cme->def->type;
536 }
537}
538
539ID
540rb_get_cme_def_body_attr_id(const rb_callable_method_entry_t *cme)
541{
542 return cme->def->body.attr.id;
543}
544
545ID rb_get_symbol_id(VALUE namep);
546
547enum method_optimized_type
548rb_get_cme_def_body_optimized_type(const rb_callable_method_entry_t *cme)
549{
550 return cme->def->body.optimized.type;
551}
552
553unsigned int
554rb_get_cme_def_body_optimized_index(const rb_callable_method_entry_t *cme)
555{
556 return cme->def->body.optimized.index;
557}
558
560rb_get_cme_def_body_cfunc(const rb_callable_method_entry_t *cme)
561{
562 return UNALIGNED_MEMBER_PTR(cme->def, body.cfunc);
563}
564
565uintptr_t
566rb_get_def_method_serial(const rb_method_definition_t *def)
567{
568 return def->method_serial;
569}
570
571ID
572rb_get_def_original_id(const rb_method_definition_t *def)
573{
574 return def->original_id;
575}
576
577int
578rb_get_mct_argc(const rb_method_cfunc_t *mct)
579{
580 return mct->argc;
581}
582
583void *
584rb_get_mct_func(const rb_method_cfunc_t *mct)
585{
586 return (void*)mct->func; // this field is defined as type VALUE (*func)(ANYARGS)
587}
588
589const rb_iseq_t *
590rb_get_def_iseq_ptr(rb_method_definition_t *def)
591{
592 return def_iseq_ptr(def);
593}
594
595VALUE
596rb_get_def_bmethod_proc(rb_method_definition_t *def)
597{
598 RUBY_ASSERT(def->type == VM_METHOD_TYPE_BMETHOD);
599 return def->body.bmethod.proc;
600}
601
602const rb_iseq_t *
603rb_get_iseq_body_local_iseq(const rb_iseq_t *iseq)
604{
605 return iseq->body->local_iseq;
606}
607
608const rb_iseq_t *
609rb_get_iseq_body_parent_iseq(const rb_iseq_t *iseq)
610{
611 return iseq->body->parent_iseq;
612}
613
614unsigned int
615rb_get_iseq_body_local_table_size(const rb_iseq_t *iseq)
616{
617 return iseq->body->local_table_size;
618}
619
620VALUE *
621rb_get_iseq_body_iseq_encoded(const rb_iseq_t *iseq)
622{
623 return iseq->body->iseq_encoded;
624}
625
626unsigned
627rb_get_iseq_body_stack_max(const rb_iseq_t *iseq)
628{
629 return iseq->body->stack_max;
630}
631
632bool
633rb_get_iseq_flags_has_lead(const rb_iseq_t *iseq)
634{
635 return iseq->body->param.flags.has_lead;
636}
637
638bool
639rb_get_iseq_flags_has_opt(const rb_iseq_t *iseq)
640{
641 return iseq->body->param.flags.has_opt;
642}
643
644bool
645rb_get_iseq_flags_has_kw(const rb_iseq_t *iseq)
646{
647 return iseq->body->param.flags.has_kw;
648}
649
650bool
651rb_get_iseq_flags_has_post(const rb_iseq_t *iseq)
652{
653 return iseq->body->param.flags.has_post;
654}
655
656bool
657rb_get_iseq_flags_has_kwrest(const rb_iseq_t *iseq)
658{
659 return iseq->body->param.flags.has_kwrest;
660}
661
662bool
663rb_get_iseq_flags_has_rest(const rb_iseq_t *iseq)
664{
665 return iseq->body->param.flags.has_rest;
666}
667
668bool
669rb_get_iseq_flags_ruby2_keywords(const rb_iseq_t *iseq)
670{
671 return iseq->body->param.flags.ruby2_keywords;
672}
673
674bool
675rb_get_iseq_flags_has_block(const rb_iseq_t *iseq)
676{
677 return iseq->body->param.flags.has_block;
678}
679
680bool
681rb_get_iseq_flags_ambiguous_param0(const rb_iseq_t *iseq)
682{
683 return iseq->body->param.flags.ambiguous_param0;
684}
685
686bool
687rb_get_iseq_flags_accepts_no_kwarg(const rb_iseq_t *iseq)
688{
689 return iseq->body->param.flags.accepts_no_kwarg;
690}
691
692const rb_seq_param_keyword_struct *
693rb_get_iseq_body_param_keyword(const rb_iseq_t *iseq)
694{
695 return iseq->body->param.keyword;
696}
697
698unsigned
699rb_get_iseq_body_param_size(const rb_iseq_t *iseq)
700{
701 return iseq->body->param.size;
702}
703
704int
705rb_get_iseq_body_param_lead_num(const rb_iseq_t *iseq)
706{
707 return iseq->body->param.lead_num;
708}
709
710int
711rb_get_iseq_body_param_opt_num(const rb_iseq_t *iseq)
712{
713 return iseq->body->param.opt_num;
714}
715
716const VALUE *
717rb_get_iseq_body_param_opt_table(const rb_iseq_t *iseq)
718{
719 return iseq->body->param.opt_table;
720}
721
722VALUE
723rb_optimized_call(VALUE *recv, rb_execution_context_t *ec, int argc, VALUE *argv, int kw_splat, VALUE block_handler)
724{
725 rb_proc_t *proc;
726 GetProcPtr(recv, proc);
727 return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
728}
729
730unsigned int
731rb_yjit_iseq_builtin_attrs(const rb_iseq_t *iseq)
732{
733 return iseq->body->builtin_attrs;
734}
735
736// If true, the iseq has only opt_invokebuiltin_delegate_leave and leave insns.
737static bool
738invokebuiltin_delegate_leave_p(const rb_iseq_t *iseq)
739{
740 unsigned int invokebuiltin_len = insn_len(BIN(opt_invokebuiltin_delegate_leave));
741 unsigned int leave_len = insn_len(BIN(leave));
742 return iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
743 rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[0]) == BIN(opt_invokebuiltin_delegate_leave) &&
744 rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave);
745}
746
747// Return an rb_builtin_function if the iseq contains only that builtin function.
748const struct rb_builtin_function *
749rb_yjit_builtin_function(const rb_iseq_t *iseq)
750{
751 if (invokebuiltin_delegate_leave_p(iseq)) {
752 return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1];
753 }
754 else {
755 return NULL;
756 }
757}
758
759VALUE
760rb_yjit_str_simple_append(VALUE str1, VALUE str2)
761{
762 return rb_str_cat(str1, RSTRING_PTR(str2), RSTRING_LEN(str2));
763}
764
766rb_get_ec_cfp(const rb_execution_context_t *ec)
767{
768 return ec->cfp;
769}
770
771const rb_iseq_t *
772rb_get_cfp_iseq(struct rb_control_frame_struct *cfp)
773{
774 return cfp->iseq;
775}
776
777VALUE *
778rb_get_cfp_pc(struct rb_control_frame_struct *cfp)
779{
780 return (VALUE*)cfp->pc;
781}
782
783VALUE *
784rb_get_cfp_sp(struct rb_control_frame_struct *cfp)
785{
786 return cfp->sp;
787}
788
789void
790rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc)
791{
792 cfp->pc = pc;
793}
794
795void
796rb_set_cfp_sp(struct rb_control_frame_struct *cfp, VALUE *sp)
797{
798 cfp->sp = sp;
799}
800
801VALUE
802rb_get_cfp_self(struct rb_control_frame_struct *cfp)
803{
804 return cfp->self;
805}
806
807VALUE *
808rb_get_cfp_ep(struct rb_control_frame_struct *cfp)
809{
810 return (VALUE*)cfp->ep;
811}
812
813const VALUE *
814rb_get_cfp_ep_level(struct rb_control_frame_struct *cfp, uint32_t lv)
815{
816 uint32_t i;
817 const VALUE *ep = (VALUE*)cfp->ep;
818 for (i = 0; i < lv; i++) {
819 ep = VM_ENV_PREV_EP(ep);
820 }
821 return ep;
822}
823
824extern VALUE *rb_vm_base_ptr(struct rb_control_frame_struct *cfp);
825
826VALUE
827rb_yarv_class_of(VALUE obj)
828{
829 return rb_class_of(obj);
830}
831
832// YJIT needs this function to never allocate and never raise
833VALUE
834rb_yarv_str_eql_internal(VALUE str1, VALUE str2)
835{
836 // We wrap this since it's static inline
837 return rb_str_eql_internal(str1, str2);
838}
839
840VALUE
841rb_str_neq_internal(VALUE str1, VALUE str2)
842{
843 return rb_str_eql_internal(str1, str2) == Qtrue ? Qfalse : Qtrue;
844}
845
846// YJIT needs this function to never allocate and never raise
847VALUE
848rb_yarv_ary_entry_internal(VALUE ary, long offset)
849{
850 return rb_ary_entry_internal(ary, offset);
851}
852
853extern VALUE rb_ary_unshift_m(int argc, VALUE *argv, VALUE ary);
854
855VALUE
856rb_yjit_rb_ary_subseq_length(VALUE ary, long beg)
857{
858 long len = RARRAY_LEN(ary);
859 return rb_ary_subseq(ary, beg, len);
860}
861
862VALUE
863rb_yjit_fix_div_fix(VALUE recv, VALUE obj)
864{
865 return rb_fix_div_fix(recv, obj);
866}
867
868VALUE
869rb_yjit_fix_mod_fix(VALUE recv, VALUE obj)
870{
871 return rb_fix_mod_fix(recv, obj);
872}
873
874// Return non-zero when `obj` is an array and its last item is a
875// `ruby2_keywords` hash. We don't support this kind of splat.
876size_t
877rb_yjit_ruby2_keywords_splat_p(VALUE obj)
878{
879 if (!RB_TYPE_P(obj, T_ARRAY)) return 0;
880 long len = RARRAY_LEN(obj);
881 if (len == 0) return 0;
882 VALUE last = RARRAY_AREF(obj, len - 1);
883 if (!RB_TYPE_P(last, T_HASH)) return 0;
884 return FL_TEST_RAW(last, RHASH_PASS_AS_KEYWORDS);
885}
886
887// Print the Ruby source location of some ISEQ for debugging purposes
888void
889rb_yjit_dump_iseq_loc(const rb_iseq_t *iseq, uint32_t insn_idx)
890{
891 char *ptr;
892 long len;
893 VALUE path = rb_iseq_path(iseq);
894 RSTRING_GETMEM(path, ptr, len);
895 fprintf(stderr, "%s %.*s:%u\n", __func__, (int)len, ptr, rb_iseq_line_no(iseq, insn_idx));
896}
897
898// The FL_TEST() macro
899VALUE
900rb_FL_TEST(VALUE obj, VALUE flags)
901{
902 return RB_FL_TEST(obj, flags);
903}
904
905// The FL_TEST_RAW() macro, normally an internal implementation detail
906VALUE
907rb_FL_TEST_RAW(VALUE obj, VALUE flags)
908{
909 return FL_TEST_RAW(obj, flags);
910}
911
912// The RB_TYPE_P macro
913bool
914rb_RB_TYPE_P(VALUE obj, enum ruby_value_type t)
915{
916 return RB_TYPE_P(obj, t);
917}
918
919long
920rb_RSTRUCT_LEN(VALUE st)
921{
922 return RSTRUCT_LEN(st);
923}
924
925// There are RSTRUCT_SETs in ruby/internal/core/rstruct.h and internal/struct.h
926// with different types (int vs long) for k. Here we use the one from ruby/internal/core/rstruct.h,
927// which takes an int.
928void
929rb_RSTRUCT_SET(VALUE st, int k, VALUE v)
930{
931 RSTRUCT_SET(st, k, v);
932}
933
934const struct rb_callinfo *
935rb_get_call_data_ci(const struct rb_call_data *cd)
936{
937 return cd->ci;
938}
939
940bool
941rb_BASIC_OP_UNREDEFINED_P(enum ruby_basic_operators bop, uint32_t klass)
942{
943 return BASIC_OP_UNREDEFINED_P(bop, klass);
944}
945
946VALUE
947rb_RCLASS_ORIGIN(VALUE c)
948{
949 return RCLASS_ORIGIN(c);
950}
951
952// Return the string encoding index
953int
954rb_ENCODING_GET(VALUE obj)
955{
956 return RB_ENCODING_GET(obj);
957}
958
959bool
960rb_yjit_multi_ractor_p(void)
961{
962 return rb_multi_ractor_p();
963}
964
965// For debug builds
966void
967rb_assert_iseq_handle(VALUE handle)
968{
969 RUBY_ASSERT_ALWAYS(rb_objspace_markable_object_p(handle));
970 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle, imemo_iseq));
971}
972
973int
974rb_IMEMO_TYPE_P(VALUE imemo, enum imemo_type imemo_type)
975{
976 return IMEMO_TYPE_P(imemo, imemo_type);
977}
978
979void
980rb_assert_cme_handle(VALUE handle)
981{
982 RUBY_ASSERT_ALWAYS(rb_objspace_markable_object_p(handle));
983 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle, imemo_ment));
984}
985
986// Used for passing a callback and other data over rb_objspace_each_objects
988 rb_iseq_callback callback;
989 void *data;
990};
991
992// Heap-walking callback for rb_yjit_for_each_iseq().
993static int
994for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data)
995{
996 const struct iseq_callback_data *callback_data = (struct iseq_callback_data *)data;
997 VALUE v = (VALUE)vstart;
998 for (; v != (VALUE)vend; v += stride) {
999 void *ptr = asan_poisoned_object_p(v);
1000 asan_unpoison_object(v, false);
1001
1002 if (rb_obj_is_iseq(v)) {
1003 rb_iseq_t *iseq = (rb_iseq_t *)v;
1004 callback_data->callback(iseq, callback_data->data);
1005 }
1006
1007 asan_poison_object_if(ptr, v);
1008 }
1009 return 0;
1010}
1011
1012// Iterate through the whole GC heap and invoke a callback for each iseq.
1013// Used for global code invalidation.
1014void
1015rb_yjit_for_each_iseq(rb_iseq_callback callback, void *data)
1016{
1017 struct iseq_callback_data callback_data = { .callback = callback, .data = data };
1018 rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data);
1019}
1020
1021// For running write barriers from Rust. Required when we add a new edge in the
1022// object graph from `old` to `young`.
1023void
1024rb_yjit_obj_written(VALUE old, VALUE young, const char *file, int line)
1025{
1026 rb_obj_written(old, Qundef, young, file, line);
1027}
1028
1029// Acquire the VM lock and then signal all other Ruby threads (ractors) to
1030// contend for the VM lock, putting them to sleep. YJIT uses this to evict
1031// threads running inside generated code so among other things, it can
1032// safely change memory protection of regions housing generated code.
1033void
1034rb_yjit_vm_lock_then_barrier(unsigned int *recursive_lock_level, const char *file, int line)
1035{
1036 rb_vm_lock_enter(recursive_lock_level, file, line);
1037 rb_vm_barrier();
1038}
1039
1040// Release the VM lock. The lock level must point to the same integer used to
1041// acquire the lock.
1042void
1043rb_yjit_vm_unlock(unsigned int *recursive_lock_level, const char *file, int line)
1044{
1045 rb_vm_lock_leave(recursive_lock_level, file, line);
1046}
1047
1048void
1049rb_yjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception)
1050{
1051 RB_VM_LOCK_ENTER();
1052 rb_vm_barrier();
1053
1054 // Compile a block version starting at the current instruction
1055 uint8_t *rb_yjit_iseq_gen_entry_point(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception); // defined in Rust
1056 uint8_t *code_ptr = rb_yjit_iseq_gen_entry_point(iseq, ec, jit_exception);
1057
1058 if (jit_exception) {
1059 iseq->body->jit_exception = (rb_jit_func_t)code_ptr;
1060 }
1061 else {
1062 iseq->body->jit_entry = (rb_jit_func_t)code_ptr;
1063 }
1064
1065 RB_VM_LOCK_LEAVE();
1066}
1067
1068// GC root for interacting with the GC
1070 bool unused; // empty structs are not legal in C99
1071};
1072
1073static void
1074yjit_root_free(void *ptr)
1075{
1076 // Do nothing. The root lives as long as the process.
1077}
1078
1079static size_t
1080yjit_root_memsize(const void *ptr)
1081{
1082 // Count off-gc-heap allocation size of the dependency table
1083 return 0; // TODO: more accurate accounting
1084}
1085
1086// GC callback during compaction
1087static void
1088yjit_root_update_references(void *ptr)
1089{
1090 // Do nothing since we use rb_gc_mark(), which pins.
1091}
1092
1093void rb_yjit_root_mark(void *ptr); // in Rust
1094
1095// Custom type for interacting with the GC
1096// TODO: make this write barrier protected
1097static const rb_data_type_t yjit_root_type = {
1098 "yjit_root",
1099 {rb_yjit_root_mark, yjit_root_free, yjit_root_memsize, yjit_root_update_references},
1100 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1101};
1102
1103// For dealing with refinements
1104void
1105rb_yjit_invalidate_all_method_lookup_assumptions(void)
1106{
1107 // It looks like Module#using actually doesn't need to invalidate all the
1108 // method caches, so we do nothing here for now.
1109}
1110
1111// Number of object shapes, which might be useful for investigating YJIT exit reasons.
1112static VALUE
1113object_shape_count(rb_execution_context_t *ec, VALUE self)
1114{
1115 // next_shape_id starts from 0, so it's the same as the count
1116 return ULONG2NUM((unsigned long)GET_SHAPE_TREE()->next_shape_id);
1117}
1118
1119// Assert that we have the VM lock. Relevant mostly for multi ractor situations.
1120// The GC takes the lock before calling us, and this asserts that it indeed happens.
1121void
1122rb_yjit_assert_holding_vm_lock(void)
1123{
1124 ASSERT_vm_locking();
1125}
1126
1127// The number of stack slots that vm_sendish() pops for send and invokesuper.
1128size_t
1129rb_yjit_sendish_sp_pops(const struct rb_callinfo *ci)
1130{
1131 return 1 - sp_inc_of_sendish(ci); // + 1 to ignore return value push
1132}
1133
1134// The number of stack slots that vm_sendish() pops for invokeblock.
1135size_t
1136rb_yjit_invokeblock_sp_pops(const struct rb_callinfo *ci)
1137{
1138 return 1 - sp_inc_of_invokeblock(ci); // + 1 to ignore return value push
1139}
1140
1141// Setup jit_return to avoid returning a non-Qundef value on a non-FINISH frame.
1142// See [jit_compile_exception] for details.
1143void
1144rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *leave_exception)
1145{
1146 if (VM_FRAME_FINISHED_P(cfp)) {
1147 // If it's a FINISH frame, just normally exit with a non-Qundef value.
1148 cfp->jit_return = leave_exit;
1149 }
1150 else if (cfp->jit_return) {
1151 while (!VM_FRAME_FINISHED_P(cfp)) {
1152 if (cfp->jit_return == leave_exit) {
1153 // Unlike jit_exec(), leave_exit is not safe on a non-FINISH frame on
1154 // jit_exec_exception(). See [jit_exec] and [jit_exec_exception] for
1155 // details. Exit to the interpreter with Qundef to let it keep executing
1156 // other Ruby frames.
1157 cfp->jit_return = leave_exception;
1158 return;
1159 }
1160 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1161 }
1162 }
1163 else {
1164 // If the caller was not JIT code, exit to the interpreter with Qundef
1165 // to keep executing Ruby frames with the interpreter.
1166 cfp->jit_return = leave_exception;
1167 }
1168}
1169
1170// Primitives used by yjit.rb
1171VALUE rb_yjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self);
1172VALUE rb_yjit_print_stats_p(rb_execution_context_t *ec, VALUE self);
1173VALUE rb_yjit_trace_exit_locations_enabled_p(rb_execution_context_t *ec, VALUE self);
1174VALUE rb_yjit_get_stats(rb_execution_context_t *ec, VALUE self, VALUE context);
1175VALUE rb_yjit_reset_stats_bang(rb_execution_context_t *ec, VALUE self);
1176VALUE rb_yjit_disasm_iseq(rb_execution_context_t *ec, VALUE self, VALUE iseq);
1177VALUE rb_yjit_insns_compiled(rb_execution_context_t *ec, VALUE self, VALUE iseq);
1178VALUE rb_yjit_code_gc(rb_execution_context_t *ec, VALUE self);
1179VALUE rb_yjit_simulate_oom_bang(rb_execution_context_t *ec, VALUE self);
1180VALUE rb_yjit_get_exit_locations(rb_execution_context_t *ec, VALUE self);
1181VALUE rb_yjit_enable(rb_execution_context_t *ec, VALUE self, VALUE gen_stats, VALUE print_stats);
1182
1183// Preprocessed yjit.rb generated during build
1184#include "yjit.rbinc"
1185
1186// Initialize the GC hooks
1187void
1188rb_yjit_init_gc_hooks(void)
1189{
1190 struct yjit_root_struct *root;
1191 VALUE yjit_root = TypedData_Make_Struct(0, struct yjit_root_struct, &yjit_root_type, root);
1192 rb_gc_register_mark_object(yjit_root);
1193}
#define RUBY_ASSERT(expr)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:177
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
Definition assert.h:167
VALUE rb_profile_frame_full_label(VALUE frame)
Identical to rb_profile_frame_label(), except it returns a qualified result.
VALUE rb_profile_frame_absolute_path(VALUE frame)
Identical to rb_profile_frame_path(), except it tries to expand the returning path.
VALUE rb_profile_frame_path(VALUE frame)
Queries the path of the passed backtrace.
VALUE rb_profile_frame_first_lineno(VALUE frame)
Queries the first line of the method of the passed frame pointer.
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:43
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:44
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
Definition fl_type.h:495
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
#define ULONG2NUM
Old name of RB_ULONG2NUM.
Definition long.h:60
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
Definition fl_type.h:132
#define Qtrue
Old name of RUBY_Qtrue.
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
#define Qfalse
Old name of RUBY_Qfalse.
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
#define NIL_P
Old name of RB_NIL_P.
static VALUE rb_class_of(VALUE obj)
Object to class mapping function.
Definition globals.h:172
static int RB_ENCODING_GET(VALUE obj)
Just another name of rb_enc_get_index.
Definition encoding.h:194
int len
Length of the buffer.
Definition io.h:8
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:51
static long rb_array_len(VALUE a)
Queries the length of the array.
Definition rarray.h:255
#define RARRAY_AREF(a, i)
Definition rarray.h:403
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
Definition rstring.h:488
#define TypedData_Make_Struct(klass, type, data_type, sval)
Identical to TypedData_Wrap_Struct, except it allocates a new data region internally instead of takin...
Definition rtypeddata.h:497
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
#define RTEST
This is an old name of RB_TEST.
#define USE_FLONUM
Ruby's ordinal objects.
Definition robject.h:83
struct RObject::@48::@49 heap
Object that use separated memory region for instance variables use this pattern.
VALUE * ivptr
Pointer to a C array that holds instance variables.
Definition robject.h:97
struct rb_id_table * iv_index_tbl
This is a table that holds instance variable name to index mapping.
Definition robject.h:107
Ruby's String.
Definition rstring.h:196
Definition method.h:62
This is the struct that holds necessary info for a struct.
Definition rtypeddata.h:200
struct rb_iseq_constant_body::@151 param
parameter information
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
ruby_value_type
C-level type of an object.
Definition value_type.h:112