Ruby 4.0.10p0 (2026-11-19 revision a704b7cd95a42cc0cf5f307137bf3aac0285b915)
thread_pthread.c
1/* -*-c-*- */
2/**********************************************************************
3
4 thread_pthread.c -
5
6 $Author$
7
8 Copyright (C) 2004-2007 Koichi Sasada
9
10**********************************************************************/
11
12#ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION
13
14#include "internal/gc.h"
15#include "internal/sanitizers.h"
16
17#ifdef HAVE_SYS_RESOURCE_H
18#include <sys/resource.h>
19#endif
20#ifdef HAVE_THR_STKSEGMENT
21#include <thread.h>
22#endif
23#if defined(HAVE_FCNTL_H)
24#include <fcntl.h>
25#elif defined(HAVE_SYS_FCNTL_H)
26#include <sys/fcntl.h>
27#endif
28#ifdef HAVE_SYS_PRCTL_H
29#include <sys/prctl.h>
30#endif
31#if defined(HAVE_SYS_TIME_H)
32#include <sys/time.h>
33#endif
34#if defined(__HAIKU__)
35#include <kernel/OS.h>
36#endif
37#ifdef __linux__
38#include <sys/syscall.h> /* for SYS_gettid */
39#endif
40#include <time.h>
41#include <signal.h>
42
43#if defined __APPLE__
44# include <AvailabilityMacros.h>
45#endif
46
47#if defined(HAVE_SYS_EVENTFD_H) && defined(HAVE_EVENTFD)
48# define USE_EVENTFD (1)
49# include <sys/eventfd.h>
50#else
51# define USE_EVENTFD (0)
52#endif
53
54#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
55 defined(CLOCK_REALTIME) && defined(CLOCK_MONOTONIC) && \
56 defined(HAVE_CLOCK_GETTIME)
57static pthread_condattr_t condattr_mono;
58static pthread_condattr_t *condattr_monotonic = &condattr_mono;
59#else
60static const void *const condattr_monotonic = NULL;
61#endif
62
63/* A retiring shared native thread frees its own context while the threads it
64 * parked are still suspended with that context as their target. */
65#define COROUTINE_TARGET_MAY_BE_FREED 1
66
67#include COROUTINE_H
68
69#ifndef HAVE_SYS_EVENT_H
70#define HAVE_SYS_EVENT_H 0
71#endif
72
73#ifndef HAVE_SYS_EPOLL_H
74#define HAVE_SYS_EPOLL_H 0
75#else
76// force setting for debug
77// #undef HAVE_SYS_EPOLL_H
78// #define HAVE_SYS_EPOLL_H 0
79#endif
80
81#ifndef USE_MN_THREADS
82 #if defined(__EMSCRIPTEN__) || defined(COROUTINE_PTHREAD_CONTEXT)
83 // on __EMSCRIPTEN__ provides epoll* declarations, but no implementations.
84 // on COROUTINE_PTHREAD_CONTEXT, it doesn't worth to use it.
85 #define USE_MN_THREADS 0
86 #elif HAVE_SYS_EPOLL_H
87 #include <sys/epoll.h>
88 #define USE_MN_THREADS 1
89 #elif HAVE_SYS_EVENT_H
90 #include <sys/event.h>
91 #define USE_MN_THREADS 1
92 #else
93 #define USE_MN_THREADS 0
94 #endif
95#endif
96
97#ifdef HAVE_SCHED_YIELD
98#define native_thread_yield() (void)sched_yield()
99#else
100#define native_thread_yield() ((void)0)
101#endif
102
103// native thread wrappers
104
105#define NATIVE_MUTEX_LOCK_DEBUG 0
106#define NATIVE_MUTEX_LOCK_DEBUG_YIELD 0
107
108static void
109mutex_debug(const char *msg, void *lock)
110{
111 if (NATIVE_MUTEX_LOCK_DEBUG) {
112 int r;
113 static pthread_mutex_t dbglock = PTHREAD_MUTEX_INITIALIZER;
114
115 if ((r = pthread_mutex_lock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
116 fprintf(stdout, "%s: %p\n", msg, lock);
117 if ((r = pthread_mutex_unlock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
118 }
119}
120
121void
122rb_native_mutex_lock(pthread_mutex_t *lock)
123{
124 int r;
125#if NATIVE_MUTEX_LOCK_DEBUG_YIELD
126 native_thread_yield();
127#endif
128 mutex_debug("lock", lock);
129 if ((r = pthread_mutex_lock(lock)) != 0) {
130 rb_bug_errno("pthread_mutex_lock", r);
131 }
132}
133
134void
135rb_native_mutex_unlock(pthread_mutex_t *lock)
136{
137 int r;
138 mutex_debug("unlock", lock);
139 if ((r = pthread_mutex_unlock(lock)) != 0) {
140 rb_bug_errno("pthread_mutex_unlock", r);
141 }
142}
143
144int
145rb_native_mutex_trylock(pthread_mutex_t *lock)
146{
147 int r;
148 mutex_debug("trylock", lock);
149 if ((r = pthread_mutex_trylock(lock)) != 0) {
150 if (r == EBUSY) {
151 return EBUSY;
152 }
153 else {
154 rb_bug_errno("pthread_mutex_trylock", r);
155 }
156 }
157 return 0;
158}
159
160void
161rb_native_mutex_initialize(pthread_mutex_t *lock)
162{
163 int r = pthread_mutex_init(lock, 0);
164 mutex_debug("init", lock);
165 if (r != 0) {
166 rb_bug_errno("pthread_mutex_init", r);
167 }
168}
169
170void
171rb_native_mutex_destroy(pthread_mutex_t *lock)
172{
173 int r = pthread_mutex_destroy(lock);
174 mutex_debug("destroy", lock);
175 if (r != 0) {
176 rb_bug_errno("pthread_mutex_destroy", r);
177 }
178}
179
180void
181rb_native_cond_initialize(rb_nativethread_cond_t *cond)
182{
183 int r = pthread_cond_init(cond, condattr_monotonic);
184 if (r != 0) {
185 rb_bug_errno("pthread_cond_init", r);
186 }
187}
188
189void
190rb_native_cond_destroy(rb_nativethread_cond_t *cond)
191{
192 int r = pthread_cond_destroy(cond);
193 if (r != 0) {
194 rb_bug_errno("pthread_cond_destroy", r);
195 }
196}
197
198/*
199 * In OS X 10.7 (Lion), pthread_cond_signal and pthread_cond_broadcast return
200 * EAGAIN after retrying 8192 times. You can see them in the following page:
201 *
202 * http://www.opensource.apple.com/source/Libc/Libc-763.11/pthreads/pthread_cond.c
203 *
204 * The following rb_native_cond_signal and rb_native_cond_broadcast functions
205 * need to retrying until pthread functions don't return EAGAIN.
206 */
207
208void
209rb_native_cond_signal(rb_nativethread_cond_t *cond)
210{
211 int r;
212 do {
213 r = pthread_cond_signal(cond);
214 } while (r == EAGAIN);
215 if (r != 0) {
216 rb_bug_errno("pthread_cond_signal", r);
217 }
218}
219
220void
221rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
222{
223 int r;
224 do {
225 r = pthread_cond_broadcast(cond);
226 } while (r == EAGAIN);
227 if (r != 0) {
228 rb_bug_errno("rb_native_cond_broadcast", r);
229 }
230}
231
232void
233rb_native_cond_wait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex)
234{
235 int r = pthread_cond_wait(cond, mutex);
236 if (r != 0) {
237 rb_bug_errno("pthread_cond_wait", r);
238 }
239}
240
241static int
242native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, const rb_hrtime_t *abs)
243{
244 int r;
245 struct timespec ts;
246
247 /*
248 * An old Linux may return EINTR. Even though POSIX says
249 * "These functions shall not return an error code of [EINTR]".
250 * http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_cond_timedwait.html
251 * Let's hide it from arch generic code.
252 */
253 do {
254 rb_hrtime2timespec(&ts, abs);
255 r = pthread_cond_timedwait(cond, mutex, &ts);
256 } while (r == EINTR);
257
258 if (r != 0 && r != ETIMEDOUT) {
259 rb_bug_errno("pthread_cond_timedwait", r);
260 }
261
262 return r;
263}
264
265static rb_hrtime_t
266native_cond_timeout(rb_nativethread_cond_t *cond, const rb_hrtime_t rel)
267{
268 if (condattr_monotonic) {
269 return rb_hrtime_add(rb_hrtime_now(), rel);
270 }
271 else {
272 struct timespec ts;
273
274 rb_timespec_now(&ts);
275 return rb_hrtime_add(rb_timespec2hrtime(&ts), rel);
276 }
277}
278
279void
280rb_native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, unsigned long msec)
281{
282 rb_hrtime_t hrmsec = native_cond_timeout(cond, RB_HRTIME_PER_MSEC * msec);
283 native_cond_timedwait(cond, mutex, &hrmsec);
284}
285
286// thread scheduling
287
288static rb_internal_thread_event_hook_t *rb_internal_thread_event_hooks = NULL;
289static void rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th);
290
291#if 0
292static const char *
293event_name(rb_event_flag_t event)
294{
295 switch (event) {
297 return "STARTED";
299 return "READY";
301 return "RESUMED";
303 return "SUSPENDED";
305 return "EXITED";
306 }
307 return "no-event";
308}
309
310#define RB_INTERNAL_THREAD_HOOK(event, th) \
311 if (UNLIKELY(rb_internal_thread_event_hooks)) { \
312 fprintf(stderr, "[thread=%"PRIxVALUE"] %s in %s (%s:%d)\n", th->self, event_name(event), __func__, __FILE__, __LINE__); \
313 rb_thread_execute_hooks(event, th); \
314 }
315#else
316#define RB_INTERNAL_THREAD_HOOK(event, th) if (UNLIKELY(rb_internal_thread_event_hooks)) { rb_thread_execute_hooks(event, th); }
317#endif
318
319static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */
320
321#if defined(SIGVTALRM) && !defined(__EMSCRIPTEN__)
322# define USE_UBF_LIST 1
323#endif
324
325static void threadptr_trap_interrupt(rb_thread_t *);
326
327static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt);
328static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt);
329static void native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th);
330
331static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r);
332static void timer_thread_wakeup(void);
333static void timer_thread_wakeup_locked(rb_vm_t *vm);
334static void timer_thread_wakeup_force(void);
335static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th);
336static void coroutine_transfer0(struct coroutine_context *transfer_from,
337 struct coroutine_context *transfer_to, bool to_dead);
338
339#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s)
340
341static bool
342th_has_dedicated_nt(const rb_thread_t *th)
343{
344 // TODO: th->has_dedicated_nt
345 return th->nt->dedicated > 0;
346}
347
349static void
350thread_sched_dump_(const char *file, int line, struct rb_thread_sched *sched)
351{
352 fprintf(stderr, "@%s:%d running:%d\n", file, line, sched->running ? (int)sched->running->serial : -1);
353 rb_thread_t *th;
354 int i = 0;
355 ccan_list_for_each(&sched->readyq, th, sched.node.readyq) {
356 i++; if (i>10) rb_bug("too many");
357 fprintf(stderr, " ready:%d (%sNT:%d)\n", th->serial,
358 th->nt ? (th->nt->dedicated ? "D" : "S") : "x",
359 th->nt ? (int)th->nt->serial : -1);
360 }
361}
362
363#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s)
364
366static void
367ractor_sched_dump_(const char *file, int line, rb_vm_t *vm)
368{
369 rb_ractor_t *r;
370
371 fprintf(stderr, "ractor_sched_dump %s:%d\n", file, line);
372
373 int i = 0;
374 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
375 i++;
376 if (i>10) rb_bug("!!");
377 fprintf(stderr, " %d ready:%d\n", i, rb_ractor_id(r));
378 }
379}
380
381#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__)
382#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__)
383
384static void
385thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th)
386{
387#if VM_CHECK_MODE > 0
388 VM_ASSERT(sched->lock_owner == NULL);
389
390 sched->lock_owner = th;
391#endif
392}
393
394static void
395thread_sched_set_unlocked(struct rb_thread_sched *sched, rb_thread_t *th)
396{
397#if VM_CHECK_MODE > 0
398 VM_ASSERT(sched->lock_owner == th);
399
400 sched->lock_owner = NULL;
401#endif
402}
403
404static void
405thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line)
406{
407 rb_native_mutex_lock(&sched->lock_);
408
409#if VM_CHECK_MODE
410 RUBY_DEBUG_LOG2(file, line, "r:%d th:%u", th ? (int)rb_ractor_id(th->ractor) : -1, rb_th_serial(th));
411#else
412 RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th));
413#endif
414
415 thread_sched_set_locked(sched, th);
416}
417
418static void
419thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line)
420{
421 RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th));
422
423 thread_sched_set_unlocked(sched, th);
424
425 rb_native_mutex_unlock(&sched->lock_);
426}
427
428static void
429ASSERT_thread_sched_locked(struct rb_thread_sched *sched, rb_thread_t *th)
430{
431 VM_ASSERT(rb_native_mutex_trylock(&sched->lock_) == EBUSY);
432
433#if VM_CHECK_MODE
434 if (th) {
435 VM_ASSERT(sched->lock_owner == th);
436 }
437 else {
438 VM_ASSERT(sched->lock_owner != NULL);
439 }
440#endif
441}
442
443#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__)
444#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__)
445
447static unsigned int
448rb_ractor_serial(const rb_ractor_t *r)
449{
450 if (r) {
451 return rb_ractor_id(r);
452 }
453 else {
454 return 0;
455 }
456}
457
458static void
459ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr)
460{
461#if VM_CHECK_MODE > 0
462 VM_ASSERT(vm->ractor.sched.lock_owner == NULL);
463 VM_ASSERT(vm->ractor.sched.locked == false);
464
465 vm->ractor.sched.lock_owner = cr;
466 vm->ractor.sched.locked = true;
467#endif
468}
469
470static void
471ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr)
472{
473#if VM_CHECK_MODE > 0
474 VM_ASSERT(vm->ractor.sched.locked);
475 VM_ASSERT(vm->ractor.sched.lock_owner == cr);
476
477 vm->ractor.sched.locked = false;
478 vm->ractor.sched.lock_owner = NULL;
479#endif
480}
481
482static void
483ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line)
484{
485 rb_native_mutex_lock(&vm->ractor.sched.lock);
486
487#if VM_CHECK_MODE
488 RUBY_DEBUG_LOG2(file, line, "cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner));
489#else
490 RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr));
491#endif
492
493 ractor_sched_set_locked(vm, cr);
494}
495
496static void
497ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line)
498{
499 RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr));
500
501 ractor_sched_set_unlocked(vm, cr);
502 rb_native_mutex_unlock(&vm->ractor.sched.lock);
503}
504
505static void
506ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr)
507{
508 VM_ASSERT(rb_native_mutex_trylock(&vm->ractor.sched.lock) == EBUSY);
509 VM_ASSERT(vm->ractor.sched.locked);
510 VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr);
511}
512
514static bool
515ractor_sched_running_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
516{
517 rb_thread_t *rth;
518 ccan_list_for_each(&vm->ractor.sched.running_threads, rth, sched.node.running_threads) {
519 if (rth == th) return true;
520 }
521 return false;
522}
523
525static unsigned int
526ractor_sched_running_threads_size(rb_vm_t *vm)
527{
528 rb_thread_t *th;
529 unsigned int i = 0;
530 ccan_list_for_each(&vm->ractor.sched.running_threads, th, sched.node.running_threads) {
531 i++;
532 }
533 return i;
534}
535
537static unsigned int
538ractor_sched_timeslice_threads_size(rb_vm_t *vm)
539{
540 rb_thread_t *th;
541 unsigned int i = 0;
542 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
543 i++;
544 }
545 return i;
546}
547
549static bool
550ractor_sched_timeslice_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
551{
552 rb_thread_t *rth;
553 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, rth, sched.node.timeslice_threads) {
554 if (rth == th) return true;
555 }
556 return false;
557}
558
559static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm);
560
561// setup timeslice signals by the timer thread.
562static void
563thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm,
564 rb_thread_t *add_th, rb_thread_t *del_th, rb_thread_t *add_timeslice_th)
565{
566#if USE_RUBY_DEBUG_LOG
567 unsigned int prev_running_cnt = vm->ractor.sched.running_cnt;
568#endif
569
570 rb_thread_t *del_timeslice_th;
571
572 if (del_th && sched->is_running_timeslice) {
573 del_timeslice_th = del_th;
574 sched->is_running_timeslice = false;
575 }
576 else {
577 del_timeslice_th = NULL;
578 }
579
580 RUBY_DEBUG_LOG("+:%u -:%u +ts:%u -ts:%u",
581 rb_th_serial(add_th), rb_th_serial(del_th),
582 rb_th_serial(add_timeslice_th), rb_th_serial(del_timeslice_th));
583
584 ractor_sched_lock(vm, cr);
585 {
586 // update running_threads
587 if (del_th) {
588 VM_ASSERT(ractor_sched_running_threads_contain_p(vm, del_th));
589 VM_ASSERT(del_timeslice_th != NULL ||
590 !ractor_sched_timeslice_threads_contain_p(vm, del_th));
591
592 ccan_list_del_init(&del_th->sched.node.running_threads);
593 vm->ractor.sched.running_cnt--;
594
595 if (UNLIKELY(vm->ractor.sched.barrier_waiting)) {
596 ractor_sched_barrier_join_signal_locked(vm);
597 }
598 sched->is_running = false;
599 }
600
601 if (add_th) {
602 if (vm->ractor.sched.barrier_waiting) {
603 // TODO: GC barrier check?
604 RUBY_DEBUG_LOG("barrier_waiting");
605 RUBY_VM_SET_VM_BARRIER_INTERRUPT(add_th->ec);
606 }
607
608 VM_ASSERT(!ractor_sched_running_threads_contain_p(vm, add_th));
609 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_th));
610
611 ccan_list_add(&vm->ractor.sched.running_threads, &add_th->sched.node.running_threads);
612 vm->ractor.sched.running_cnt++;
613 sched->is_running = true;
614 }
615
616 if (add_timeslice_th) {
617 // update timeslice threads
618 int was_empty = ccan_list_empty(&vm->ractor.sched.timeslice_threads);
619 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_timeslice_th));
620 ccan_list_add(&vm->ractor.sched.timeslice_threads, &add_timeslice_th->sched.node.timeslice_threads);
621 sched->is_running_timeslice = true;
622 if (was_empty) {
623 timer_thread_wakeup_locked(vm);
624 }
625 }
626
627 if (del_timeslice_th) {
628 VM_ASSERT(ractor_sched_timeslice_threads_contain_p(vm, del_timeslice_th));
629 ccan_list_del_init(&del_timeslice_th->sched.node.timeslice_threads);
630 }
631
632 VM_ASSERT(ractor_sched_running_threads_size(vm) == vm->ractor.sched.running_cnt);
633 VM_ASSERT(ractor_sched_timeslice_threads_size(vm) <= vm->ractor.sched.running_cnt);
634 }
635 ractor_sched_unlock(vm, cr);
636
637 //RUBY_DEBUG_LOG("+:%u -:%u +ts:%u -ts:%u run:%u->%u",
638 // rb_th_serial(add_th), rb_th_serial(del_th),
639 // rb_th_serial(add_timeslice_th), rb_th_serial(del_timeslice_th),
640 RUBY_DEBUG_LOG("run:%u->%u", prev_running_cnt, vm->ractor.sched.running_cnt);
641}
642
643static void
644thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th)
645{
646 ASSERT_thread_sched_locked(sched, th);
647 VM_ASSERT(sched->running == th);
648
649 rb_vm_t *vm = th->vm;
650 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, ccan_list_empty(&sched->readyq) ? NULL : th);
651}
652
653static void
654thread_sched_del_running_thread(struct rb_thread_sched *sched, rb_thread_t *th)
655{
656 ASSERT_thread_sched_locked(sched, th);
657
658 rb_vm_t *vm = th->vm;
659 thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th, NULL);
660}
661
662void
663rb_add_running_thread(rb_thread_t *th)
664{
665 struct rb_thread_sched *sched = TH_SCHED(th);
666
667 thread_sched_lock(sched, th);
668 {
669 thread_sched_add_running_thread(sched, th);
670 }
671 thread_sched_unlock(sched, th);
672}
673
674void
675rb_del_running_thread(rb_thread_t *th)
676{
677 struct rb_thread_sched *sched = TH_SCHED(th);
678
679 thread_sched_lock(sched, th);
680 {
681 thread_sched_del_running_thread(sched, th);
682 }
683 thread_sched_unlock(sched, th);
684}
685
686// setup current or next running thread
687// sched->running should be set only on this function.
688//
689// if th is NULL, there is no running threads.
690static void
691thread_sched_set_running(struct rb_thread_sched *sched, rb_thread_t *th)
692{
693 RUBY_DEBUG_LOG("th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th));
694 VM_ASSERT(sched->running != th);
695
696 sched->running = th;
697}
698
700static bool
701thread_sched_readyq_contain_p(struct rb_thread_sched *sched, rb_thread_t *th)
702{
703 rb_thread_t *rth;
704 ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) {
705 if (rth == th) {
706 VM_ASSERT(th->sched.node.is_ready);
707 return true;
708 }
709 }
710 VM_ASSERT(!th->sched.node.is_ready);
711 return false;
712}
713
714// deque thread from the ready queue.
715// if the ready queue is empty, return NULL.
716//
717// return deque'ed running thread (or NULL).
718static rb_thread_t *
719thread_sched_deq(struct rb_thread_sched *sched)
720{
721 ASSERT_thread_sched_locked(sched, NULL);
722 rb_thread_t *next_th;
723
724 VM_ASSERT(sched->running != NULL);
725
726 if (ccan_list_empty(&sched->readyq)) {
727 next_th = NULL;
728 }
729 else {
730 next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq);
731 VM_ASSERT(next_th->sched.node.is_ready);
732 next_th->sched.node.is_ready = false;
733
734 VM_ASSERT(sched->readyq_cnt > 0);
735 sched->readyq_cnt--;
736 ccan_list_node_init(&next_th->sched.node.readyq);
737 }
738
739 RUBY_DEBUG_LOG("next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt);
740
741 return next_th;
742}
743
744// enqueue ready thread to the ready queue.
745static void
746thread_sched_enq(struct rb_thread_sched *sched, rb_thread_t *ready_th)
747{
748 ASSERT_thread_sched_locked(sched, NULL);
749 RUBY_DEBUG_LOG("ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt);
750
751 VM_ASSERT(sched->running != NULL);
752 VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th));
753
754 if (sched->is_running) {
755 if (ccan_list_empty(&sched->readyq)) {
756 // add sched->running to timeslice
757 thread_sched_setup_running_threads(sched, ready_th->ractor, ready_th->vm, NULL, NULL, sched->running);
758 }
759 }
760 else {
761 // ractor_sched lock is needed
762 // VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(ready_th->vm, sched->running));
763 }
764
765 ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq);
766 ready_th->sched.node.is_ready = true;
767 sched->readyq_cnt++;
768}
769
770// DNT: kick condvar
771// SNT: TODO
772static void
773thread_sched_wakeup_running_thread(struct rb_thread_sched *sched, rb_thread_t *next_th, bool will_switch)
774{
775 ASSERT_thread_sched_locked(sched, NULL);
776 VM_ASSERT(sched->running == next_th);
777
778 if (next_th) {
779 if (next_th->nt) {
780 if (th_has_dedicated_nt(next_th)) {
781 RUBY_DEBUG_LOG("pinning th:%u", next_th->serial);
782 rb_native_cond_signal(&next_th->nt->cond.readyq);
783 }
784 else {
785 // TODO
786 RUBY_DEBUG_LOG("th:%u is already running.", next_th->serial);
787 }
788 }
789 else {
790 if (will_switch) {
791 RUBY_DEBUG_LOG("th:%u (do nothing)", rb_th_serial(next_th));
792 }
793 else {
794 RUBY_DEBUG_LOG("th:%u (enq)", rb_th_serial(next_th));
795 ractor_sched_enq(next_th->vm, next_th->ractor);
796 }
797 }
798 }
799 else {
800 RUBY_DEBUG_LOG("no waiting threads%s", "");
801 }
802}
803
804// waiting -> ready (locked)
805static void
806thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch)
807{
808 RUBY_DEBUG_LOG("th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt);
809
810 VM_ASSERT(sched->running != th);
811 VM_ASSERT(!thread_sched_readyq_contain_p(sched, th));
812 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_READY, th);
813
814 if (sched->running == NULL) {
815 thread_sched_set_running(sched, th);
816 if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch);
817 }
818 else {
819 thread_sched_enq(sched, th);
820 }
821}
822
823// waiting -> ready
824//
825// `th` had became "waiting" state by `thread_sched_to_waiting`
826// and `thread_sched_to_ready` enqueue `th` to the thread ready queue.
828static void
829thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th)
830{
831 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
832
833 thread_sched_lock(sched, th);
834 {
835 thread_sched_to_ready_common(sched, th, true, false);
836 }
837 thread_sched_unlock(sched, th);
838}
839
840// wait until sched->running is `th`.
841static void
842thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer)
843{
844 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
845
846 ASSERT_thread_sched_locked(sched, th);
847 VM_ASSERT(th == rb_ec_thread_ptr(rb_current_ec_noinline()));
848
849 if (th != sched->running) {
850 // TODO: This optimization should also be made to work for MN_THREADS
851 if (th->has_dedicated_nt && th == sched->runnable_hot_th && (sched->running == NULL || sched->running->has_dedicated_nt)) {
852 RUBY_DEBUG_LOG("(nt) stealing: hot-th:%u. running:%u", rb_th_serial(th), rb_th_serial(sched->running));
853
854 // If there is a thread set to run, move it back to the front of the readyq
855 if (sched->running != NULL) {
856 rb_thread_t *running = sched->running;
857 VM_ASSERT(!thread_sched_readyq_contain_p(sched, running));
858 running->sched.node.is_ready = true;
859 ccan_list_add(&sched->readyq, &running->sched.node.readyq);
860 sched->readyq_cnt++;
861 }
862
863 // Pull off the ready queue and start running.
864 if (th->sched.node.is_ready) {
865 VM_ASSERT(thread_sched_readyq_contain_p(sched, th));
866 ccan_list_del_init(&th->sched.node.readyq);
867 th->sched.node.is_ready = false;
868 sched->readyq_cnt--;
869 }
870 thread_sched_set_running(sched, th);
871 rb_ractor_thread_switch(th->ractor, th, false);
872 }
873
874 // already deleted from running threads
875 // VM_ASSERT(!ractor_sched_running_threads_contain_p(th->vm, th)); // need locking
876
877 // wait for execution right
878 rb_thread_t *next_th;
879 while((next_th = sched->running) != th) {
880 if (th_has_dedicated_nt(th)) {
881 RUBY_DEBUG_LOG("(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running));
882
883 thread_sched_set_unlocked(sched, th);
884 {
885 RUBY_DEBUG_LOG("nt:%d cond:%p", th->nt->serial, &th->nt->cond.readyq);
886 rb_native_cond_wait(&th->nt->cond.readyq, &sched->lock_);
887 }
888 thread_sched_set_locked(sched, th);
889
890 if (sched->runnable_hot_th != NULL && sched->runnable_hot_th_waiting) {
891 VM_ASSERT(sched->runnable_hot_th != th);
892 // Give the hot thread a chance to preempt, if it's actively spinning.
893 // On multicore, this reduces the rate of core-switching. On single-core it
894 // should mostly be a nop, since the other thread can't be concurrently spinning.
895 thread_sched_unlock(sched, th);
896 thread_sched_lock(sched, th);
897 }
898
899 RUBY_DEBUG_LOG("(nt) wakeup %s", sched->running == th ? "success" : "failed");
900 if (th == sched->running) {
901 rb_ractor_thread_switch(th->ractor, th, false);
902 }
903 }
904 else {
905 // search another ready thread
906 if (can_direct_transfer &&
907 (next_th = sched->running) != NULL &&
908 !next_th->nt // next_th is running or has dedicated nt
909 ) {
910
911 RUBY_DEBUG_LOG("th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th));
912
913 thread_sched_set_unlocked(sched, th);
914 {
915 rb_ractor_set_current_ec(th->ractor, NULL);
916 thread_sched_switch(th, next_th);
917 }
918 thread_sched_set_locked(sched, th);
919 }
920 else {
921 // search another ready ractor
922 struct rb_native_thread *nt = th->nt;
923 native_thread_assign(NULL, th);
924
925 RUBY_DEBUG_LOG("th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th));
926
927 thread_sched_set_unlocked(sched, th);
928 {
929 rb_ractor_set_current_ec(th->ractor, NULL);
930 coroutine_transfer0(th->sched.context, nt->nt_context, false);
931 }
932 thread_sched_set_locked(sched, th);
933 }
934
935 VM_ASSERT(rb_current_ec_noinline() == th->ec);
936 }
937 }
938
939 VM_ASSERT(th->nt != NULL);
940 VM_ASSERT(rb_current_ec_noinline() == th->ec);
941 VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none);
942
943 // add th to running threads
944 thread_sched_add_running_thread(sched, th);
945 }
946
947 // Control transfer to the current thread is now complete. The original thread
948 // cannot steal control at this point.
949 sched->runnable_hot_th = NULL;
950 sched->runnable_hot_th_waiting = 0;
951
952 // VM_ASSERT(ractor_sched_running_threads_contain_p(th->vm, th)); need locking
953 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th);
954}
955
956// waiting -> ready -> running (locked)
957static void
958thread_sched_to_running_common(struct rb_thread_sched *sched, rb_thread_t *th)
959{
960 RUBY_DEBUG_LOG("th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th));
961
962 VM_ASSERT(sched->running != th);
963 VM_ASSERT(th_has_dedicated_nt(th));
964 VM_ASSERT(GET_THREAD() == th);
965
966 native_thread_dedicated_dec(th->vm, th->ractor, th->nt);
967
968 // waiting -> ready
969 thread_sched_to_ready_common(sched, th, false, false);
970
971 if (sched->running == th) {
972 thread_sched_add_running_thread(sched, th);
973 }
974
975 // TODO: check SNT number
976 thread_sched_wait_running_turn(sched, th, false);
977}
978
979// waiting -> ready -> running
980//
981// `th` had been waiting by `thread_sched_to_waiting()`
982// and run a dedicated task (like waitpid and so on).
983// After the dedicated task, this function is called
984// to join a normal thread-scheduling.
985static void
986thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th)
987{
988 // We are reading and writing these sched fields without lock cover, but
989 // there are no correctness issues resulting from stale cache or delayed writeback.
990 // When it works, this causes the next-scheduled thread to yield the sched lock
991 // briefly so that we can grab it if we're still spinning (not descheduled yet).
992 if (sched->runnable_hot_th == th) {
993 sched->runnable_hot_th_waiting = 1;
994 }
995 thread_sched_lock(sched, th);
996 {
997 thread_sched_to_running_common(sched, th);
998 }
999 thread_sched_unlock(sched, th);
1000}
1001
1002// resume a next thread in the thread ready queue.
1003//
1004// deque next running thread from the ready thread queue and
1005// resume this thread if available.
1006//
1007// If the next therad has a dedicated native thraed, simply signal to resume.
1008// Otherwise, make the ractor ready and other nt will run the ractor and the thread.
1009static void
1010thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch)
1011{
1012 ASSERT_thread_sched_locked(sched, th);
1013
1014 VM_ASSERT(sched->running == th);
1015 VM_ASSERT(sched->running->nt != NULL);
1016
1017 rb_thread_t *next_th = thread_sched_deq(sched);
1018
1019 RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th));
1020 VM_ASSERT(th != next_th);
1021
1022 thread_sched_set_running(sched, next_th);
1023 VM_ASSERT(next_th == sched->running);
1024 thread_sched_wakeup_running_thread(sched, next_th, will_switch);
1025
1026 if (th != next_th) {
1027 thread_sched_del_running_thread(sched, th);
1028 }
1029}
1030
1031// running -> dead (locked)
1032static void
1033thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th)
1034{
1035 RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated);
1036
1037 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1038
1039 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
1040
1041 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th);
1042}
1043
1044// running -> dead
1045static void
1046thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th)
1047{
1048 thread_sched_lock(sched, th);
1049 {
1050 thread_sched_to_dead_common(sched, th);
1051 }
1052 thread_sched_unlock(sched, th);
1053}
1054
1055// running -> waiting (locked)
1056//
1057// This thread will run dedicated task (th->nt->dedicated++).
1058static void
1059thread_sched_to_waiting_common(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately)
1060{
1061 RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated);
1062
1063 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1064
1065 native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
1066 if (!yield_immediately) {
1067 sched->runnable_hot_th = th;
1068 sched->runnable_hot_th_waiting = 0;
1069 }
1070 thread_sched_wakeup_next_thread(sched, th, false);
1071}
1072
1073// running -> waiting
1074//
1075// This thread will run a dedicated task.
1076static void
1077thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately)
1078{
1079 thread_sched_lock(sched, th);
1080 {
1081 thread_sched_to_waiting_common(sched, th, yield_immediately);
1082 }
1083 thread_sched_unlock(sched, th);
1084}
1085
1086// mini utility func
1087// return true if any there are any interrupts
1088static bool
1089ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg)
1090{
1091 VM_ASSERT(func != NULL);
1092
1093 retry:
1094 if (RUBY_VM_INTERRUPTED(th->ec)) {
1095 RUBY_DEBUG_LOG("interrupted:0x%x", th->ec->interrupt_flag);
1096 return true;
1097 }
1098
1099 rb_native_mutex_lock(&th->interrupt_lock);
1100 {
1101 if (!th->ec->raised_flag && RUBY_VM_INTERRUPTED(th->ec)) {
1102 rb_native_mutex_unlock(&th->interrupt_lock);
1103 goto retry;
1104 }
1105
1106 VM_ASSERT(th->unblock.func == NULL);
1107 th->unblock.func = func;
1108 th->unblock.arg = arg;
1109 }
1110 rb_native_mutex_unlock(&th->interrupt_lock);
1111
1112 return false;
1113}
1114
1115static void
1116ubf_clear(rb_thread_t *th)
1117{
1118 if (th->unblock.func) {
1119 rb_native_mutex_lock(&th->interrupt_lock);
1120 {
1121 th->unblock.func = NULL;
1122 th->unblock.arg = NULL;
1123 }
1124 rb_native_mutex_unlock(&th->interrupt_lock);
1125 }
1126}
1127
1128static void
1129ubf_waiting(void *ptr)
1130{
1131 rb_thread_t *th = (rb_thread_t *)ptr;
1132 struct rb_thread_sched *sched = TH_SCHED(th);
1133
1134 // only once. it is safe because th->interrupt_lock is already acquired.
1135 th->unblock.func = NULL;
1136 th->unblock.arg = NULL;
1137
1138 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
1139
1140 thread_sched_lock(sched, th);
1141 {
1142 if (sched->running == th) {
1143 // not sleeping yet.
1144 }
1145 else {
1146 thread_sched_to_ready_common(sched, th, true, false);
1147 }
1148 }
1149 thread_sched_unlock(sched, th);
1150}
1151
1152// running -> waiting
1153//
1154// This thread will sleep until other thread wakeup the thread.
1155static void
1156thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th)
1157{
1158 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
1159
1160 RB_VM_SAVE_MACHINE_CONTEXT(th);
1161
1162
1163 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1164
1165 thread_sched_lock(sched, th);
1166 {
1167 // NOTE: there's a lock ordering inversion here with the ubf call, but it's benign.
1168 if (ubf_set(th, ubf_waiting, (void *)th)) {
1169 RUBY_DEBUG_LOG("th:%u interrupted", rb_th_serial(th));
1170 }
1171 else {
1172 bool can_direct_transfer = !th_has_dedicated_nt(th);
1173 // NOTE: th->status is set before and after this sleep outside of this function in `sleep_forever`
1174 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1175 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1176 }
1177 }
1178 thread_sched_unlock(sched, th);
1179
1180 ubf_clear(th);
1181}
1182
1183// run another thread in the ready queue.
1184// continue to run if there are no ready threads.
1185static void
1186thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th)
1187{
1188 RUBY_DEBUG_LOG("th:%d sched->readyq_cnt:%d", (int)th->serial, sched->readyq_cnt);
1189
1190 thread_sched_lock(sched, th);
1191 {
1192 if (!ccan_list_empty(&sched->readyq)) {
1193 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1194 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
1195 bool can_direct_transfer = !th_has_dedicated_nt(th);
1196 thread_sched_to_ready_common(sched, th, false, can_direct_transfer);
1197 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1198 th->status = THREAD_RUNNABLE;
1199 }
1200 else {
1201 VM_ASSERT(sched->readyq_cnt == 0);
1202 }
1203 }
1204 thread_sched_unlock(sched, th);
1205}
1206
1207void
1208rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork)
1209{
1210 rb_native_mutex_initialize(&sched->lock_);
1211
1212#if VM_CHECK_MODE
1213 sched->lock_owner = NULL;
1214#endif
1215
1216 ccan_list_head_init(&sched->readyq);
1217 sched->readyq_cnt = 0;
1218
1219#if USE_MN_THREADS
1220 if (!atfork) sched->enable_mn_threads = true; // MN is enabled on Ractors
1221#endif
1222}
1223
1224static void
1225coroutine_transfer0(struct coroutine_context *transfer_from, struct coroutine_context *transfer_to, bool to_dead)
1226{
1227#ifdef RUBY_ASAN_ENABLED
1228 void **fake_stack = to_dead ? NULL : &transfer_from->fake_stack;
1229 __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size);
1230#endif
1231
1233 struct coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to);
1234
1235 /* if to_dead was passed, the caller is promising that this coroutine is finished and it should
1236 * never be resumed! */
1237 VM_ASSERT(!to_dead);
1238#ifdef RUBY_ASAN_ENABLED
1239 __sanitizer_finish_switch_fiber(transfer_from->fake_stack,
1240 (const void**)&returning_from->stack_base, &returning_from->stack_size);
1241#endif
1242
1243}
1244
1245static void
1246thread_sched_switch0(struct coroutine_context *current_cont, rb_thread_t *next_th, struct rb_native_thread *nt, bool to_dead)
1247{
1248 VM_ASSERT(!nt->dedicated);
1249 VM_ASSERT(next_th->nt == NULL);
1250
1251 RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th));
1252
1253 ruby_thread_set_native(next_th);
1254 native_thread_assign(nt, next_th);
1255
1256 coroutine_transfer0(current_cont, next_th->sched.context, to_dead);
1257}
1258
1259static void
1260thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th)
1261{
1262 struct rb_native_thread *nt = cth->nt;
1263 native_thread_assign(NULL, cth);
1264 RUBY_DEBUG_LOG("th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial);
1265 thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED);
1266}
1267
1268#if VM_CHECK_MODE > 0
1270static unsigned int
1271grq_size(rb_vm_t *vm, rb_ractor_t *cr)
1272{
1273 ASSERT_ractor_sched_locked(vm, cr);
1274
1275 rb_ractor_t *r, *prev_r = NULL;
1276 unsigned int i = 0;
1277
1278 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
1279 i++;
1280
1281 VM_ASSERT(r != prev_r);
1282 prev_r = r;
1283 }
1284 return i;
1285}
1286#endif
1287
1288static void
1289ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r)
1290{
1291 struct rb_thread_sched *sched = &r->threads.sched;
1292 rb_ractor_t *cr = NULL; // timer thread can call this function
1293
1294 VM_ASSERT(sched->running != NULL);
1295 VM_ASSERT(sched->running->nt == NULL);
1296
1297 ractor_sched_lock(vm, cr);
1298 {
1299#if VM_CHECK_MODE > 0
1300 // check if grq contains r
1301 rb_ractor_t *tr;
1302 ccan_list_for_each(&vm->ractor.sched.grq, tr, threads.sched.grq_node) {
1303 VM_ASSERT(r != tr);
1304 }
1305#endif
1306
1307 ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node);
1308 vm->ractor.sched.grq_cnt++;
1309 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1310
1311 RUBY_DEBUG_LOG("r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt);
1312
1313 rb_native_cond_signal(&vm->ractor.sched.cond);
1314
1315 // ractor_sched_dump(vm);
1316 }
1317 ractor_sched_unlock(vm, cr);
1318}
1319
1320
1321#ifndef SNT_KEEP_SECONDS
1322#define SNT_KEEP_SECONDS 0
1323#endif
1324
1325#ifndef MINIMUM_SNT
1326// make at least MINIMUM_SNT snts for debug.
1327#define MINIMUM_SNT 0
1328#endif
1329
1330static rb_ractor_t *
1331ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr)
1332{
1333 rb_ractor_t *r;
1334
1335 ractor_sched_lock(vm, cr);
1336 {
1337 RUBY_DEBUG_LOG("empty? %d", ccan_list_empty(&vm->ractor.sched.grq));
1338 // ractor_sched_dump(vm);
1339
1340 VM_ASSERT(rb_current_execution_context(false) == NULL);
1341 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1342
1343 while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) {
1344 RUBY_DEBUG_LOG("wait grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1345
1346#if SNT_KEEP_SECONDS > 0
1347 rb_hrtime_t abs = rb_hrtime_add(rb_hrtime_now(), RB_HRTIME_PER_SEC * SNT_KEEP_SECONDS);
1348 if (native_cond_timedwait(&vm->ractor.sched.cond, &vm->ractor.sched.lock, &abs) == ETIMEDOUT) {
1349 RUBY_DEBUG_LOG("timeout, grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1350 VM_ASSERT(r == NULL);
1351 vm->ractor.sched.snt_cnt--;
1352 vm->ractor.sched.running_cnt--;
1353 break;
1354 }
1355 else {
1356 RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1357 }
1358#else
1359 ractor_sched_set_unlocked(vm, cr);
1360 rb_native_cond_wait(&vm->ractor.sched.cond, &vm->ractor.sched.lock);
1361 ractor_sched_set_locked(vm, cr);
1362
1363 RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1364#endif
1365 }
1366
1367 VM_ASSERT(rb_current_execution_context(false) == NULL);
1368
1369 if (r) {
1370 VM_ASSERT(vm->ractor.sched.grq_cnt > 0);
1371 vm->ractor.sched.grq_cnt--;
1372 RUBY_DEBUG_LOG("r:%d grq_cnt:%u", (int)rb_ractor_id(r), vm->ractor.sched.grq_cnt);
1373 }
1374 else {
1375 VM_ASSERT(SNT_KEEP_SECONDS > 0);
1376 // timeout
1377 }
1378 }
1379 ractor_sched_unlock(vm, cr);
1380
1381 return r;
1382}
1383
1384void rb_ractor_lock_self(rb_ractor_t *r);
1385void rb_ractor_unlock_self(rb_ractor_t *r);
1386
1387// The current thread for a ractor is put to "sleep" (descheduled in the STOPPED_FOREVER state) waiting for
1388// a ractor action to wake it up.
1389void
1390rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ubf_arg)
1391{
1392 // ractor lock of cr is acquired
1393
1394 RUBY_DEBUG_LOG("start%s", "");
1395
1396 rb_thread_t * volatile th = rb_ec_thread_ptr(ec);
1397 struct rb_thread_sched *sched = TH_SCHED(th);
1398
1399 if (ubf_set(th, ubf, ubf_arg)) {
1400 // interrupted
1401 return;
1402 }
1403
1404 thread_sched_lock(sched, th);
1405 rb_ractor_unlock_self(cr);
1406 {
1407 // setup sleep
1408 bool can_direct_transfer = !th_has_dedicated_nt(th);
1409 RB_VM_SAVE_MACHINE_CONTEXT(th);
1410 th->status = THREAD_STOPPED_FOREVER;
1411 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1412 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1413 // sleep
1414 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1415 th->status = THREAD_RUNNABLE;
1416 }
1417 thread_sched_unlock(sched, th);
1418 rb_ractor_lock_self(cr);
1419
1420 ubf_clear(th);
1421
1422 RUBY_DEBUG_LOG("end%s", "");
1423}
1424
1425void
1426rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *r_th)
1427{
1428 // ractor lock of r is NOT acquired
1429 struct rb_thread_sched *sched = TH_SCHED(r_th);
1430
1431 RUBY_DEBUG_LOG("r:%u th:%d", (unsigned int)rb_ractor_id(r), r_th->serial);
1432
1433 thread_sched_lock(sched, r_th);
1434 {
1435 if (r_th->status == THREAD_STOPPED_FOREVER) {
1436 thread_sched_to_ready_common(sched, r_th, true, false);
1437 }
1438 }
1439 thread_sched_unlock(sched, r_th);
1440}
1441
1442static bool
1443ractor_sched_barrier_completed_p(rb_vm_t *vm)
1444{
1445 RUBY_DEBUG_LOG("run:%u wait:%u", vm->ractor.sched.running_cnt, vm->ractor.sched.barrier_waiting_cnt);
1446 VM_ASSERT(vm->ractor.sched.running_cnt - 1 >= vm->ractor.sched.barrier_waiting_cnt);
1447
1448 return (vm->ractor.sched.running_cnt - vm->ractor.sched.barrier_waiting_cnt) == 1;
1449}
1450
1451void
1452rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr)
1453{
1454 VM_ASSERT(cr == GET_RACTOR());
1455 VM_ASSERT(vm->ractor.sync.lock_owner == cr); // VM is locked
1456 VM_ASSERT(!vm->ractor.sched.barrier_waiting);
1457 VM_ASSERT(vm->ractor.sched.barrier_waiting_cnt == 0);
1458 VM_ASSERT(vm->ractor.sched.barrier_ractor == NULL);
1459 VM_ASSERT(vm->ractor.sched.barrier_lock_rec == 0);
1460
1461 RUBY_DEBUG_LOG("start serial:%u", vm->ractor.sched.barrier_serial);
1462
1463 unsigned int lock_rec;
1464
1465 ractor_sched_lock(vm, cr);
1466 {
1467 vm->ractor.sched.barrier_waiting = true;
1468 vm->ractor.sched.barrier_ractor = cr;
1469 vm->ractor.sched.barrier_lock_rec = vm->ractor.sync.lock_rec;
1470
1471 // release VM lock
1472 lock_rec = vm->ractor.sync.lock_rec;
1473 vm->ractor.sync.lock_rec = 0;
1474 vm->ractor.sync.lock_owner = NULL;
1475 rb_native_mutex_unlock(&vm->ractor.sync.lock);
1476
1477 // interrupts all running threads
1478 rb_thread_t *ith;
1479 ccan_list_for_each(&vm->ractor.sched.running_threads, ith, sched.node.running_threads) {
1480 if (ith->ractor != cr) {
1481 RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith));
1482 RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec);
1483 }
1484 }
1485
1486 // wait for other ractors
1487 while (!ractor_sched_barrier_completed_p(vm)) {
1488 ractor_sched_set_unlocked(vm, cr);
1489 rb_native_cond_wait(&vm->ractor.sched.barrier_complete_cond, &vm->ractor.sched.lock);
1490 ractor_sched_set_locked(vm, cr);
1491 }
1492
1493 RUBY_DEBUG_LOG("completed seirial:%u", vm->ractor.sched.barrier_serial);
1494
1495 // no other ractors are there
1496 vm->ractor.sched.barrier_serial++;
1497 vm->ractor.sched.barrier_waiting_cnt = 0;
1498 rb_native_cond_broadcast(&vm->ractor.sched.barrier_release_cond);
1499
1500 // acquire VM lock
1501 rb_native_mutex_lock(&vm->ractor.sync.lock);
1502 vm->ractor.sync.lock_rec = lock_rec;
1503 vm->ractor.sync.lock_owner = cr;
1504 }
1505
1506 // do not release ractor_sched_lock and there is no newly added (resumed) thread
1507 // thread_sched_setup_running_threads
1508}
1509
1510// called from vm_lock_leave if the vm_lock used for barrierred
1511void
1512rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr)
1513{
1514 RUBY_DEBUG_LOG("serial:%u", (unsigned int)vm->ractor.sched.barrier_serial - 1);
1515 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1516 VM_ASSERT(vm->ractor.sched.barrier_ractor);
1517 VM_ASSERT(vm->ractor.sched.barrier_lock_rec > 0);
1518
1519 vm->ractor.sched.barrier_waiting = false;
1520 vm->ractor.sched.barrier_ractor = NULL;
1521 vm->ractor.sched.barrier_lock_rec = 0;
1522 ractor_sched_unlock(vm, cr);
1523}
1524
1525static void
1526ractor_sched_barrier_join_signal_locked(rb_vm_t *vm)
1527{
1528 if (ractor_sched_barrier_completed_p(vm)) {
1529 rb_native_cond_signal(&vm->ractor.sched.barrier_complete_cond);
1530 }
1531}
1532
1533static void
1534ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th)
1535{
1536 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1537
1538 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1539
1540 while (vm->ractor.sched.barrier_serial == barrier_serial) {
1541 RUBY_DEBUG_LOG("sleep serial:%u", barrier_serial);
1542 RB_VM_SAVE_MACHINE_CONTEXT(th);
1543
1544 rb_ractor_t *cr = th->ractor;
1545 ractor_sched_set_unlocked(vm, cr);
1546 rb_native_cond_wait(&vm->ractor.sched.barrier_release_cond, &vm->ractor.sched.lock);
1547 ractor_sched_set_locked(vm, cr);
1548
1549 RUBY_DEBUG_LOG("wakeup serial:%u", barrier_serial);
1550 }
1551}
1552
1553void
1554rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr)
1555{
1556 VM_ASSERT(cr->threads.sched.running != NULL); // running ractor
1557 VM_ASSERT(cr == GET_RACTOR());
1558 VM_ASSERT(vm->ractor.sync.lock_owner == NULL); // VM is locked, but owner == NULL
1559 VM_ASSERT(vm->ractor.sched.barrier_waiting); // VM needs barrier sync
1560
1561#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0
1562 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1563#endif
1564
1565 RUBY_DEBUG_LOG("join");
1566
1567 rb_native_mutex_unlock(&vm->ractor.sync.lock);
1568 {
1569 VM_ASSERT(vm->ractor.sched.barrier_waiting); // VM needs barrier sync
1570 VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial);
1571
1572 ractor_sched_lock(vm, cr);
1573 {
1574 // running_cnt
1575 vm->ractor.sched.barrier_waiting_cnt++;
1576 RUBY_DEBUG_LOG("waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_waiting_cnt, barrier_serial);
1577
1578 ractor_sched_barrier_join_signal_locked(vm);
1579 ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running);
1580 }
1581 ractor_sched_unlock(vm, cr);
1582 }
1583
1584 rb_native_mutex_lock(&vm->ractor.sync.lock);
1585 // VM locked here
1586}
1587
1588#if 0
1589// TODO
1590
1591static void clear_thread_cache_altstack(void);
1592
1593static void
1594rb_thread_sched_destroy(struct rb_thread_sched *sched)
1595{
1596 /*
1597 * only called once at VM shutdown (not atfork), another thread
1598 * may still grab vm->gvl.lock when calling gvl_release at
1599 * the end of thread_start_func_2
1600 */
1601 if (0) {
1602 rb_native_mutex_destroy(&sched->lock);
1603 }
1604 clear_thread_cache_altstack();
1605}
1606#endif
1607
1608#ifdef RB_THREAD_T_HAS_NATIVE_ID
1609static int
1610get_native_thread_id(void)
1611{
1612#ifdef __linux__
1613 return (int)syscall(SYS_gettid);
1614#elif defined(__FreeBSD__)
1615 return pthread_getthreadid_np();
1616#endif
1617}
1618#endif
1619
1620#if defined(HAVE_WORKING_FORK)
1621void rb_internal_thread_event_hooks_rw_lock_atfork(void);
1622
1623static void
1624thread_sched_atfork(struct rb_thread_sched *sched)
1625{
1626 current_fork_gen++;
1627 rb_thread_sched_init(sched, true);
1628 rb_thread_t *th = GET_THREAD();
1629 rb_vm_t *vm = GET_VM();
1630
1631 if (th_has_dedicated_nt(th)) {
1632 vm->ractor.sched.snt_cnt = 0;
1633 }
1634 else {
1635 vm->ractor.sched.snt_cnt = 1;
1636 }
1637 vm->ractor.sched.running_cnt = 0;
1638
1639 rb_native_mutex_initialize(&vm->ractor.sched.lock);
1640#if VM_CHECK_MODE > 0
1641 vm->ractor.sched.lock_owner = NULL;
1642 vm->ractor.sched.locked = false;
1643#endif
1644
1645 // rb_native_cond_destroy(&vm->ractor.sched.cond);
1646 rb_native_cond_initialize(&vm->ractor.sched.cond);
1647 rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond);
1648 rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond);
1649
1650 ccan_list_head_init(&vm->ractor.sched.grq);
1651 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1652 ccan_list_head_init(&vm->ractor.sched.running_threads);
1653
1654 rb_internal_thread_event_hooks_rw_lock_atfork();
1655
1656 VM_ASSERT(sched->is_running);
1657 sched->is_running_timeslice = false;
1658
1659 if (sched->running != th) {
1660 thread_sched_to_running(sched, th);
1661 }
1662 else {
1663 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, NULL);
1664 }
1665
1666#ifdef RB_THREAD_T_HAS_NATIVE_ID
1667 if (th->nt) {
1668 th->nt->tid = get_native_thread_id();
1669 }
1670#endif
1671}
1672
1673#endif
1674
1675#ifdef RB_THREAD_LOCAL_SPECIFIER
1676static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread;
1677#else
1678static pthread_key_t ruby_native_thread_key;
1679#endif
1680
1681static void
1682null_func(int i)
1683{
1684 /* null */
1685 // This function can be called from signal handler
1686 // RUBY_DEBUG_LOG("i:%d", i);
1687}
1688
1689rb_thread_t *
1690ruby_thread_from_native(void)
1691{
1692#ifdef RB_THREAD_LOCAL_SPECIFIER
1693 return ruby_native_thread;
1694#else
1695 return pthread_getspecific(ruby_native_thread_key);
1696#endif
1697}
1698
1699int
1700ruby_thread_set_native(rb_thread_t *th)
1701{
1702 if (th) {
1703#ifdef USE_UBF_LIST
1704 ccan_list_node_init(&th->sched.node.ubf);
1705#endif
1706 }
1707
1708 // setup TLS
1709
1710 if (th && th->ec) {
1711 rb_ractor_set_current_ec(th->ractor, th->ec);
1712 }
1713#ifdef RB_THREAD_LOCAL_SPECIFIER
1714 ruby_native_thread = th;
1715 return 1;
1716#else
1717 return pthread_setspecific(ruby_native_thread_key, th) == 0;
1718#endif
1719}
1720
1721static void native_thread_setup(struct rb_native_thread *nt);
1722static void native_thread_setup_on_thread(struct rb_native_thread *nt);
1723
1724void
1725Init_native_thread(rb_thread_t *main_th)
1726{
1727#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
1728 if (condattr_monotonic) {
1729 int r = pthread_condattr_init(condattr_monotonic);
1730 if (r == 0) {
1731 r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC);
1732 }
1733 if (r) condattr_monotonic = NULL;
1734 }
1735#endif
1736
1737#ifndef RB_THREAD_LOCAL_SPECIFIER
1738 if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) {
1739 rb_bug("pthread_key_create failed (ruby_native_thread_key)");
1740 }
1741 if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) {
1742 rb_bug("pthread_key_create failed (ruby_current_ec_key)");
1743 }
1744#endif
1745 ruby_posix_signal(SIGVTALRM, null_func);
1746
1747 // setup vm
1748 rb_vm_t *vm = main_th->vm;
1749 rb_native_mutex_initialize(&vm->ractor.sched.lock);
1750 rb_native_cond_initialize(&vm->ractor.sched.cond);
1751 rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond);
1752 rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond);
1753
1754 ccan_list_head_init(&vm->ractor.sched.grq);
1755 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1756 ccan_list_head_init(&vm->ractor.sched.running_threads);
1757
1758 // setup main thread
1759 main_th->nt->thread_id = pthread_self();
1760 main_th->nt->serial = 1;
1761#ifdef RUBY_NT_SERIAL
1762 ruby_nt_serial = 1;
1763#endif
1764 ruby_thread_set_native(main_th);
1765 native_thread_setup(main_th->nt);
1766 native_thread_setup_on_thread(main_th->nt);
1767
1768 TH_SCHED(main_th)->running = main_th;
1769 main_th->has_dedicated_nt = 1;
1770
1771 thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL, NULL);
1772
1773 // setup main NT
1774 main_th->nt->dedicated = 1;
1775 main_th->nt->vm = vm;
1776
1777 // setup mn
1778 vm->ractor.sched.dnt_cnt = 1;
1779}
1780
1781extern int ruby_mn_threads_enabled;
1782
1783void
1784ruby_mn_threads_params(void)
1785{
1786 rb_vm_t *vm = GET_VM();
1787 rb_ractor_t *main_ractor = GET_RACTOR();
1788
1789 const char *mn_threads_cstr = getenv("RUBY_MN_THREADS");
1790 bool enable_mn_threads = false;
1791
1792 if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) {
1793 // enabled
1794 ruby_mn_threads_enabled = 1;
1795 }
1796 main_ractor->threads.sched.enable_mn_threads = enable_mn_threads;
1797
1798 const char *max_cpu_cstr = getenv("RUBY_MAX_CPU");
1799 const int default_max_cpu = 8; // TODO: CPU num?
1800 int max_cpu = default_max_cpu;
1801
1802 if (USE_MN_THREADS && max_cpu_cstr) {
1803 int given_max_cpu = atoi(max_cpu_cstr);
1804 if (given_max_cpu > 0) {
1805 max_cpu = given_max_cpu;
1806 }
1807 }
1808
1809 vm->ractor.sched.max_cpu = max_cpu;
1810}
1811
1812static void
1813native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt)
1814{
1815 RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1);
1816
1817 if (nt->dedicated == 0) {
1818 ractor_sched_lock(vm, cr);
1819 {
1820 vm->ractor.sched.snt_cnt--;
1821 vm->ractor.sched.dnt_cnt++;
1822 }
1823 ractor_sched_unlock(vm, cr);
1824 }
1825
1826 nt->dedicated++;
1827}
1828
1829static void
1830native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt)
1831{
1832 RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1);
1833 VM_ASSERT(nt->dedicated > 0);
1834 nt->dedicated--;
1835
1836 if (nt->dedicated == 0) {
1837 ractor_sched_lock(vm, cr);
1838 {
1839 nt->vm->ractor.sched.snt_cnt++;
1840 nt->vm->ractor.sched.dnt_cnt--;
1841 }
1842 ractor_sched_unlock(vm, cr);
1843 }
1844}
1845
1846static void
1847native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th)
1848{
1849#if USE_RUBY_DEBUG_LOG
1850 if (nt) {
1851 if (th->nt) {
1852 RUBY_DEBUG_LOG("th:%d nt:%d->%d", (int)th->serial, (int)th->nt->serial, (int)nt->serial);
1853 }
1854 else {
1855 RUBY_DEBUG_LOG("th:%d nt:NULL->%d", (int)th->serial, (int)nt->serial);
1856 }
1857 }
1858 else {
1859 if (th->nt) {
1860 RUBY_DEBUG_LOG("th:%d nt:%d->NULL", (int)th->serial, (int)th->nt->serial);
1861 }
1862 else {
1863 RUBY_DEBUG_LOG("th:%d nt:NULL->NULL", (int)th->serial);
1864 }
1865 }
1866#endif
1867
1868 th->nt = nt;
1869}
1870
1871static void
1872native_thread_destroy_atfork(struct rb_native_thread *nt)
1873{
1874 if (nt) {
1875 /* We can't call rb_native_cond_destroy here because according to the
1876 * specs of pthread_cond_destroy:
1877 *
1878 * Attempting to destroy a condition variable upon which other threads
1879 * are currently blocked results in undefined behavior.
1880 *
1881 * Specifically, glibc's pthread_cond_destroy waits on all the other
1882 * listeners. Since after forking all the threads are dead, the condition
1883 * variable's listeners will never wake up, so it will hang forever.
1884 */
1885
1886 RB_ALTSTACK_FREE(nt->altstack);
1887 ruby_xfree(nt->nt_context);
1888 ruby_xfree(nt);
1889 }
1890}
1891
1892static void
1893native_thread_destroy(struct rb_native_thread *nt)
1894{
1895 if (nt) {
1896 rb_native_cond_destroy(&nt->cond.readyq);
1897
1898 if (&nt->cond.readyq != &nt->cond.intr) {
1899 rb_native_cond_destroy(&nt->cond.intr);
1900 }
1901
1902 native_thread_destroy_atfork(nt);
1903 }
1904}
1905
1906#if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP
1907#define STACKADDR_AVAILABLE 1
1908#elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP
1909#define STACKADDR_AVAILABLE 1
1910#undef MAINSTACKADDR_AVAILABLE
1911#define MAINSTACKADDR_AVAILABLE 1
1912void *pthread_get_stackaddr_np(pthread_t);
1913size_t pthread_get_stacksize_np(pthread_t);
1914#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1915#define STACKADDR_AVAILABLE 1
1916#elif defined HAVE_PTHREAD_GETTHRDS_NP
1917#define STACKADDR_AVAILABLE 1
1918#elif defined __HAIKU__
1919#define STACKADDR_AVAILABLE 1
1920#endif
1921
1922#ifndef MAINSTACKADDR_AVAILABLE
1923# ifdef STACKADDR_AVAILABLE
1924# define MAINSTACKADDR_AVAILABLE 1
1925# else
1926# define MAINSTACKADDR_AVAILABLE 0
1927# endif
1928#endif
1929#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack)
1930# define get_main_stack(addr, size) get_stack(addr, size)
1931#endif
1932
1933#ifdef STACKADDR_AVAILABLE
1934/*
1935 * Get the initial address and size of current thread's stack
1936 */
1937static int
1938get_stack(void **addr, size_t *size)
1939{
1940#define CHECK_ERR(expr) \
1941 {int err = (expr); if (err) return err;}
1942#ifdef HAVE_PTHREAD_GETATTR_NP /* Linux */
1943 pthread_attr_t attr;
1944 size_t guard = 0;
1945 STACK_GROW_DIR_DETECTION;
1946 CHECK_ERR(pthread_getattr_np(pthread_self(), &attr));
1947# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1948 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1949 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1950# else
1951 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1952 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1953# endif
1954# ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE
1955 CHECK_ERR(pthread_attr_getguardsize(&attr, &guard));
1956# else
1957 guard = getpagesize();
1958# endif
1959 *size -= guard;
1960 pthread_attr_destroy(&attr);
1961#elif defined HAVE_PTHREAD_ATTR_GET_NP /* FreeBSD, DragonFly BSD, NetBSD */
1962 pthread_attr_t attr;
1963 CHECK_ERR(pthread_attr_init(&attr));
1964 CHECK_ERR(pthread_attr_get_np(pthread_self(), &attr));
1965# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1966 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1967# else
1968 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1969 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1970# endif
1971 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1972 pthread_attr_destroy(&attr);
1973#elif (defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP) /* MacOS X */
1974 pthread_t th = pthread_self();
1975 *addr = pthread_get_stackaddr_np(th);
1976 *size = pthread_get_stacksize_np(th);
1977#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1978 stack_t stk;
1979# if defined HAVE_THR_STKSEGMENT /* Solaris */
1980 CHECK_ERR(thr_stksegment(&stk));
1981# else /* OpenBSD */
1982 CHECK_ERR(pthread_stackseg_np(pthread_self(), &stk));
1983# endif
1984 *addr = stk.ss_sp;
1985 *size = stk.ss_size;
1986#elif defined HAVE_PTHREAD_GETTHRDS_NP /* AIX */
1987 pthread_t th = pthread_self();
1988 struct __pthrdsinfo thinfo;
1989 char reg[256];
1990 int regsiz=sizeof(reg);
1991 CHECK_ERR(pthread_getthrds_np(&th, PTHRDSINFO_QUERY_ALL,
1992 &thinfo, sizeof(thinfo),
1993 &reg, &regsiz));
1994 *addr = thinfo.__pi_stackaddr;
1995 /* Must not use thinfo.__pi_stacksize for size.
1996 It is around 3KB smaller than the correct size
1997 calculated by thinfo.__pi_stackend - thinfo.__pi_stackaddr. */
1998 *size = thinfo.__pi_stackend - thinfo.__pi_stackaddr;
1999 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
2000#elif defined __HAIKU__
2001 thread_info info;
2002 STACK_GROW_DIR_DETECTION;
2003 CHECK_ERR(get_thread_info(find_thread(NULL), &info));
2004 *addr = info.stack_base;
2005 *size = (uintptr_t)info.stack_end - (uintptr_t)info.stack_base;
2006 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
2007#else
2008#error STACKADDR_AVAILABLE is defined but not implemented.
2009#endif
2010 return 0;
2011#undef CHECK_ERR
2012}
2013#endif
2014
2015static struct {
2016 rb_nativethread_id_t id;
2017 size_t stack_maxsize;
2018 VALUE *stack_start;
2019} native_main_thread;
2020
2021#ifdef STACK_END_ADDRESS
2022extern void *STACK_END_ADDRESS;
2023#endif
2024
2025enum {
2026 RUBY_STACK_SPACE_LIMIT = 1024 * 1024, /* 1024KB */
2027 RUBY_STACK_SPACE_RATIO = 5
2028};
2029
2030static size_t
2031space_size(size_t stack_size)
2032{
2033 size_t space_size = stack_size / RUBY_STACK_SPACE_RATIO;
2034 if (space_size > RUBY_STACK_SPACE_LIMIT) {
2035 return RUBY_STACK_SPACE_LIMIT;
2036 }
2037 else {
2038 return space_size;
2039 }
2040}
2041
2042static void
2043native_thread_init_main_thread_stack(void *addr)
2044{
2045 native_main_thread.id = pthread_self();
2046#ifdef RUBY_ASAN_ENABLED
2047 addr = asan_get_real_stack_addr((void *)addr);
2048#endif
2049
2050#if MAINSTACKADDR_AVAILABLE
2051 if (native_main_thread.stack_maxsize) return;
2052 {
2053 void* stackaddr;
2054 size_t size;
2055 if (get_main_stack(&stackaddr, &size) == 0) {
2056 native_main_thread.stack_maxsize = size;
2057 native_main_thread.stack_start = stackaddr;
2058 goto bound_check;
2059 }
2060 }
2061#endif
2062#ifdef STACK_END_ADDRESS
2063 native_main_thread.stack_start = STACK_END_ADDRESS;
2064#else
2065 if (!native_main_thread.stack_start ||
2066 STACK_UPPER((VALUE *)(void *)&addr,
2067 native_main_thread.stack_start > (VALUE *)addr,
2068 native_main_thread.stack_start < (VALUE *)addr)) {
2069 native_main_thread.stack_start = (VALUE *)addr;
2070 }
2071#endif
2072 {
2073#if defined(HAVE_GETRLIMIT)
2074#if defined(PTHREAD_STACK_DEFAULT)
2075# if PTHREAD_STACK_DEFAULT < RUBY_STACK_SPACE*5
2076# error "PTHREAD_STACK_DEFAULT is too small"
2077# endif
2078 size_t size = PTHREAD_STACK_DEFAULT;
2079#else
2080 size_t size = RUBY_VM_THREAD_VM_STACK_SIZE;
2081#endif
2082 size_t space;
2083 int pagesize = getpagesize();
2084 struct rlimit rlim;
2085 STACK_GROW_DIR_DETECTION;
2086 if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
2087 size = (size_t)rlim.rlim_cur;
2088 }
2089 addr = native_main_thread.stack_start;
2090 if (IS_STACK_DIR_UPPER()) {
2091 space = ((size_t)((char *)addr + size) / pagesize) * pagesize - (size_t)addr;
2092 }
2093 else {
2094 space = (size_t)addr - ((size_t)((char *)addr - size) / pagesize + 1) * pagesize;
2095 }
2096 native_main_thread.stack_maxsize = space;
2097#endif
2098 }
2099
2100#if MAINSTACKADDR_AVAILABLE
2101 bound_check:
2102#endif
2103 /* If addr is out of range of main-thread stack range estimation, */
2104 /* it should be on co-routine (alternative stack). [Feature #2294] */
2105 {
2106 void *start, *end;
2107 STACK_GROW_DIR_DETECTION;
2108
2109 if (IS_STACK_DIR_UPPER()) {
2110 start = native_main_thread.stack_start;
2111 end = (char *)native_main_thread.stack_start + native_main_thread.stack_maxsize;
2112 }
2113 else {
2114 start = (char *)native_main_thread.stack_start - native_main_thread.stack_maxsize;
2115 end = native_main_thread.stack_start;
2116 }
2117
2118 if ((void *)addr < start || (void *)addr > end) {
2119 /* out of range */
2120 native_main_thread.stack_start = (VALUE *)addr;
2121 native_main_thread.stack_maxsize = 0; /* unknown */
2122 }
2123 }
2124}
2125
2126#define CHECK_ERR(expr) \
2127 {int err = (expr); if (err) {rb_bug_errno(#expr, err);}}
2128
2129static int
2130native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame)
2131{
2132 rb_nativethread_id_t curr = pthread_self();
2133#ifdef RUBY_ASAN_ENABLED
2134 local_in_parent_frame = asan_get_real_stack_addr(local_in_parent_frame);
2135 th->ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
2136#endif
2137
2138 if (!native_main_thread.id) {
2139 /* This thread is the first thread, must be the main thread -
2140 * configure the native_main_thread object */
2141 native_thread_init_main_thread_stack(local_in_parent_frame);
2142 }
2143
2144 if (pthread_equal(curr, native_main_thread.id)) {
2145 th->ec->machine.stack_start = native_main_thread.stack_start;
2146 th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize;
2147 }
2148 else {
2149#ifdef STACKADDR_AVAILABLE
2150 if (th_has_dedicated_nt(th)) {
2151 void *start;
2152 size_t size;
2153
2154 if (get_stack(&start, &size) == 0) {
2155 uintptr_t diff = (uintptr_t)start - (uintptr_t)local_in_parent_frame;
2156 th->ec->machine.stack_start = local_in_parent_frame;
2157 th->ec->machine.stack_maxsize = size - diff;
2158 }
2159 }
2160#else
2161 rb_raise(rb_eNotImpError, "ruby engine can initialize only in the main thread");
2162#endif
2163 }
2164
2165 return 0;
2166}
2167
2168struct nt_param {
2169 rb_vm_t *vm;
2170 struct rb_native_thread *nt;
2171};
2172
2173static void *
2174nt_start(void *ptr);
2175
2176static int
2177native_thread_create0(struct rb_native_thread *nt)
2178{
2179 int err = 0;
2180 pthread_attr_t attr;
2181
2182 const size_t stack_size = nt->vm->default_params.thread_machine_stack_size;
2183 const size_t space = space_size(stack_size);
2184
2185 nt->machine_stack_maxsize = stack_size - space;
2186
2187#ifdef USE_SIGALTSTACK
2188 nt->altstack = rb_allocate_sigaltstack();
2189#endif
2190
2191 CHECK_ERR(pthread_attr_init(&attr));
2192
2193# ifdef PTHREAD_STACK_MIN
2194 RUBY_DEBUG_LOG("stack size: %lu", (unsigned long)stack_size);
2195 CHECK_ERR(pthread_attr_setstacksize(&attr, stack_size));
2196# endif
2197
2198# ifdef HAVE_PTHREAD_ATTR_SETINHERITSCHED
2199 CHECK_ERR(pthread_attr_setinheritsched(&attr, PTHREAD_INHERIT_SCHED));
2200# endif
2201 CHECK_ERR(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
2202
2203 err = pthread_create(&nt->thread_id, &attr, nt_start, nt);
2204
2205 RUBY_DEBUG_LOG("nt:%d err:%d", (int)nt->serial, err);
2206
2207 CHECK_ERR(pthread_attr_destroy(&attr));
2208
2209 return err;
2210}
2211
2212static void
2213native_thread_setup(struct rb_native_thread *nt)
2214{
2215 // init cond
2216 rb_native_cond_initialize(&nt->cond.readyq);
2217
2218 if (&nt->cond.readyq != &nt->cond.intr) {
2219 rb_native_cond_initialize(&nt->cond.intr);
2220 }
2221}
2222
2223static void
2224native_thread_setup_on_thread(struct rb_native_thread *nt)
2225{
2226 // init tid
2227#ifdef RB_THREAD_T_HAS_NATIVE_ID
2228 nt->tid = get_native_thread_id();
2229#endif
2230
2231 // init signal handler
2232 RB_ALTSTACK_INIT(nt->altstack, nt->altstack);
2233}
2234
2235static struct rb_native_thread *
2236native_thread_alloc(void)
2237{
2238 struct rb_native_thread *nt = ZALLOC(struct rb_native_thread);
2239 native_thread_setup(nt);
2240
2241#if USE_MN_THREADS
2242 nt->nt_context = ruby_xmalloc(sizeof(struct coroutine_context));
2243#endif
2244
2245#if USE_RUBY_DEBUG_LOG
2246 static rb_atomic_t nt_serial = 2;
2247 nt->serial = RUBY_ATOMIC_FETCH_ADD(nt_serial, 1);
2248#endif
2249 return nt;
2250}
2251
2252static int
2253native_thread_create_dedicated(rb_thread_t *th)
2254{
2255 th->nt = native_thread_alloc();
2256 th->nt->vm = th->vm;
2257 th->nt->running_thread = th;
2258 th->nt->dedicated = 1;
2259
2260 // vm stack
2261 size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE);
2262 void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE));
2263 th->sched.malloc_stack = true;
2264 rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size);
2265 th->sched.context_stack = vm_stack;
2266
2267
2268 int err = native_thread_create0(th->nt);
2269 if (!err) {
2270 // setup
2271 thread_sched_to_ready(TH_SCHED(th), th);
2272 }
2273 return err;
2274}
2275
2276static void
2277call_thread_start_func_2(rb_thread_t *th)
2278{
2279 /* Capture the address of a local in this stack frame to mark the beginning of the
2280 machine stack for this thread. This is required even if we can tell the real
2281 stack beginning from the pthread API in native_thread_init_stack, because
2282 glibc stores some of its own data on the stack before calling into user code
2283 on a new thread, and replacing that data on fiber-switch would break it (see
2284 bug #13887) */
2285 VALUE stack_start = 0;
2286 VALUE *stack_start_addr = asan_get_real_stack_addr(&stack_start);
2287
2288 native_thread_init_stack(th, stack_start_addr);
2289 thread_start_func_2(th, th->ec->machine.stack_start);
2290}
2291
2292static void *
2293nt_start(void *ptr)
2294{
2295 struct rb_native_thread *nt = (struct rb_native_thread *)ptr;
2296 rb_vm_t *vm = nt->vm;
2297
2298 native_thread_setup_on_thread(nt);
2299
2300 // init tid
2301#ifdef RB_THREAD_T_HAS_NATIVE_ID
2302 nt->tid = get_native_thread_id();
2303#endif
2304
2305#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL)
2306 ruby_nt_serial = nt->serial;
2307#endif
2308
2309 RUBY_DEBUG_LOG("nt:%u", nt->serial);
2310
2311 if (!nt->dedicated) {
2312 coroutine_initialize_main(nt->nt_context);
2313 }
2314
2315 while (1) {
2316 if (nt->dedicated) {
2317 // wait running turn
2318 rb_thread_t *th = nt->running_thread;
2319 struct rb_thread_sched *sched = TH_SCHED(th);
2320
2321 RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th));
2322 ruby_thread_set_native(th);
2323
2324 thread_sched_lock(sched, th);
2325 {
2326 if (sched->running == th) {
2327 thread_sched_add_running_thread(sched, th);
2328 }
2329 thread_sched_wait_running_turn(sched, th, false);
2330 }
2331 thread_sched_unlock(sched, th);
2332
2333 // start threads
2334 call_thread_start_func_2(th);
2335 break; // TODO: allow to change to the SNT
2336 }
2337 else {
2338 RUBY_DEBUG_LOG("check next");
2339 rb_ractor_t *r = ractor_sched_deq(vm, NULL);
2340
2341 if (r) {
2342 struct rb_thread_sched *sched = &r->threads.sched;
2343
2344 thread_sched_lock(sched, NULL);
2345 {
2346 rb_thread_t *next_th = sched->running;
2347
2348 if (next_th && next_th->nt == NULL) {
2349 RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial);
2350 thread_sched_switch0(nt->nt_context, next_th, nt, false);
2351 }
2352 else {
2353 RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th);
2354 }
2355 }
2356 thread_sched_unlock(sched, NULL);
2357 }
2358 else {
2359 // timeout -> deleted.
2360 break;
2361 }
2362
2363 if (nt->dedicated) {
2364 // SNT becomes DNT while running
2365 break;
2366 }
2367 }
2368 }
2369
2370 return NULL;
2371}
2372
2373static int native_thread_create_shared(rb_thread_t *th);
2374
2375#if USE_MN_THREADS
2376static void nt_free_stack(void *mstack);
2377#endif
2378
2379void
2380rb_threadptr_remove(rb_thread_t *th)
2381{
2382#if USE_MN_THREADS
2383 if (th->sched.malloc_stack) {
2384 // dedicated
2385 return;
2386 }
2387 else {
2388 rb_vm_t *vm = th->vm;
2389 th->sched.finished = false;
2390
2391 RB_VM_LOCKING() {
2392 ccan_list_add(&vm->ractor.sched.zombie_threads, &th->sched.node.zombie_threads);
2393 }
2394 }
2395#endif
2396}
2397
2398void
2399rb_threadptr_sched_free(rb_thread_t *th)
2400{
2401#if USE_MN_THREADS
2402 if (th->sched.malloc_stack) {
2403 // has dedicated
2404 ruby_xfree(th->sched.context_stack);
2405 native_thread_destroy(th->nt);
2406 }
2407 else {
2408 nt_free_stack(th->sched.context_stack);
2409 // TODO: how to free nt and nt->altstack?
2410 }
2411
2412 ruby_xfree(th->sched.context);
2413 th->sched.context = NULL;
2414 // VM_ASSERT(th->sched.context == NULL);
2415#else
2416 ruby_xfree(th->sched.context_stack);
2417 native_thread_destroy(th->nt);
2418#endif
2419
2420 th->nt = NULL;
2421}
2422
2423void
2424rb_thread_sched_mark_zombies(rb_vm_t *vm)
2425{
2426 if (!ccan_list_empty(&vm->ractor.sched.zombie_threads)) {
2427 rb_thread_t *zombie_th, *next_zombie_th;
2428 ccan_list_for_each_safe(&vm->ractor.sched.zombie_threads, zombie_th, next_zombie_th, sched.node.zombie_threads) {
2429 if (zombie_th->sched.finished) {
2430 ccan_list_del_init(&zombie_th->sched.node.zombie_threads);
2431 }
2432 else {
2433 rb_gc_mark(zombie_th->self);
2434 }
2435 }
2436 }
2437}
2438
2439static int
2440native_thread_create(rb_thread_t *th)
2441{
2442 VM_ASSERT(th->nt == 0);
2443 RUBY_DEBUG_LOG("th:%d has_dnt:%d", th->serial, th->has_dedicated_nt);
2444 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th);
2445
2446 if (!th->ractor->threads.sched.enable_mn_threads) {
2447 th->has_dedicated_nt = 1;
2448 }
2449
2450 if (th->has_dedicated_nt) {
2451 return native_thread_create_dedicated(th);
2452 }
2453 else {
2454 return native_thread_create_shared(th);
2455 }
2456}
2457
2458#if USE_NATIVE_THREAD_PRIORITY
2459
2460static void
2461native_thread_apply_priority(rb_thread_t *th)
2462{
2463#if defined(_POSIX_PRIORITY_SCHEDULING) && (_POSIX_PRIORITY_SCHEDULING > 0)
2464 struct sched_param sp;
2465 int policy;
2466 int priority = 0 - th->priority;
2467 int max, min;
2468 pthread_getschedparam(th->nt->thread_id, &policy, &sp);
2469 max = sched_get_priority_max(policy);
2470 min = sched_get_priority_min(policy);
2471
2472 if (min > priority) {
2473 priority = min;
2474 }
2475 else if (max < priority) {
2476 priority = max;
2477 }
2478
2479 sp.sched_priority = priority;
2480 pthread_setschedparam(th->nt->thread_id, policy, &sp);
2481#else
2482 /* not touched */
2483#endif
2484}
2485
2486#endif /* USE_NATIVE_THREAD_PRIORITY */
2487
2488static int
2489native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *exceptfds, struct timeval *timeout, rb_thread_t *th)
2490{
2491 return rb_fd_select(n, readfds, writefds, exceptfds, timeout);
2492}
2493
2494static void
2495ubf_pthread_cond_signal(void *ptr)
2496{
2497 rb_thread_t *th = (rb_thread_t *)ptr;
2498 RUBY_DEBUG_LOG("th:%u on nt:%d", rb_th_serial(th), (int)th->nt->serial);
2499 rb_native_cond_signal(&th->nt->cond.intr);
2500}
2501
2502static void
2503native_cond_sleep(rb_thread_t *th, rb_hrtime_t *rel)
2504{
2505 rb_nativethread_lock_t *lock = &th->interrupt_lock;
2506 rb_nativethread_cond_t *cond = &th->nt->cond.intr;
2507
2508 /* Solaris cond_timedwait() return EINVAL if an argument is greater than
2509 * current_time + 100,000,000. So cut up to 100,000,000. This is
2510 * considered as a kind of spurious wakeup. The caller to native_sleep
2511 * should care about spurious wakeup.
2512 *
2513 * See also [Bug #1341] [ruby-core:29702]
2514 * http://download.oracle.com/docs/cd/E19683-01/816-0216/6m6ngupgv/index.html
2515 */
2516 const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC;
2517
2518 THREAD_BLOCKING_BEGIN(th);
2519 {
2521 th->unblock.func = ubf_pthread_cond_signal;
2522 th->unblock.arg = th;
2523
2524 if (RUBY_VM_INTERRUPTED(th->ec)) {
2525 /* interrupted. return immediate */
2526 RUBY_DEBUG_LOG("interrupted before sleep th:%u", rb_th_serial(th));
2527 }
2528 else {
2529 if (!rel) {
2530 rb_native_cond_wait(cond, lock);
2531 }
2532 else {
2533 rb_hrtime_t end;
2534
2535 if (*rel > max) {
2536 *rel = max;
2537 }
2538
2539 end = native_cond_timeout(cond, *rel);
2540 native_cond_timedwait(cond, lock, &end);
2541 }
2542 }
2543 th->unblock.func = 0;
2544
2546 }
2547 THREAD_BLOCKING_END(th);
2548
2549 RUBY_DEBUG_LOG("done th:%u", rb_th_serial(th));
2550}
2551
2552#ifdef USE_UBF_LIST
2553static CCAN_LIST_HEAD(ubf_list_head);
2554static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT;
2555
2556static void
2557ubf_list_atfork(void)
2558{
2559 ccan_list_head_init(&ubf_list_head);
2560 rb_native_mutex_initialize(&ubf_list_lock);
2561}
2562
2564static bool
2565ubf_list_contain_p(rb_thread_t *th)
2566{
2567 rb_thread_t *list_th;
2568 ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) {
2569 if (list_th == th) return true;
2570 }
2571 return false;
2572}
2573
2574/* The thread 'th' is registered to be trying unblock. */
2575static void
2576register_ubf_list(rb_thread_t *th)
2577{
2578 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
2579 struct ccan_list_node *node = &th->sched.node.ubf;
2580
2581 VM_ASSERT(th->unblock.func != NULL);
2582
2583 rb_native_mutex_lock(&ubf_list_lock);
2584 {
2585 // check not connected yet
2586 if (ccan_list_empty((struct ccan_list_head*)node)) {
2587 VM_ASSERT(!ubf_list_contain_p(th));
2588 ccan_list_add(&ubf_list_head, node);
2589 }
2590 }
2591 rb_native_mutex_unlock(&ubf_list_lock);
2592
2593 timer_thread_wakeup();
2594}
2595
2596/* The thread 'th' is unblocked. It no longer need to be registered. */
2597static void
2598unregister_ubf_list(rb_thread_t *th)
2599{
2600 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
2601 struct ccan_list_node *node = &th->sched.node.ubf;
2602
2603 /* we can't allow re-entry into ubf_list_head */
2604 VM_ASSERT(th->unblock.func == NULL);
2605
2606 if (!ccan_list_empty((struct ccan_list_head*)node)) {
2607 rb_native_mutex_lock(&ubf_list_lock);
2608 {
2609 VM_ASSERT(ubf_list_contain_p(th));
2610 ccan_list_del_init(node);
2611 }
2612 rb_native_mutex_unlock(&ubf_list_lock);
2613 }
2614}
2615
2616/*
2617 * send a signal to intent that a target thread return from blocking syscall.
2618 * Maybe any signal is ok, but we chose SIGVTALRM.
2619 */
2620static void
2621ubf_wakeup_thread(rb_thread_t *th)
2622{
2623 RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id);
2624
2625 pthread_kill(th->nt->thread_id, SIGVTALRM);
2626}
2627
2628static void
2629ubf_select(void *ptr)
2630{
2631 rb_thread_t *th = (rb_thread_t *)ptr;
2632 RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(th));
2633 ubf_wakeup_thread(th);
2634 register_ubf_list(th);
2635}
2636
2637static bool
2638ubf_threads_empty(void)
2639{
2640 return ccan_list_empty(&ubf_list_head) != 0;
2641}
2642
2643static void
2644ubf_wakeup_all_threads(void)
2645{
2646 rb_thread_t *th;
2647 rb_native_mutex_lock(&ubf_list_lock);
2648 {
2649 ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) {
2650 ubf_wakeup_thread(th);
2651 }
2652 }
2653 rb_native_mutex_unlock(&ubf_list_lock);
2654}
2655
2656#else /* USE_UBF_LIST */
2657#define register_ubf_list(th) (void)(th)
2658#define unregister_ubf_list(th) (void)(th)
2659#define ubf_select 0
2660static void ubf_wakeup_all_threads(void) { return; }
2661static bool ubf_threads_empty(void) { return true; }
2662#define ubf_list_atfork() do {} while (0)
2663#endif /* USE_UBF_LIST */
2664
2665#define TT_DEBUG 0
2666#define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0)
2667
2668void
2669rb_thread_wakeup_timer_thread(int sig)
2670{
2671 // This function can be called from signal handlers so that
2672 // pthread_mutex_lock() should not be used.
2673
2674 // wakeup timer thread
2675 timer_thread_wakeup_force();
2676
2677 // interrupt main thread if main thread is available
2678 if (RUBY_ATOMIC_LOAD(system_working)) {
2679 rb_vm_t *vm = GET_VM();
2680 rb_thread_t *main_th = vm->ractor.main_thread;
2681
2682 if (main_th) {
2683 volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec);
2684
2685 if (main_th_ec) {
2686 RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec);
2687
2688 if (vm->ubf_async_safe && main_th->unblock.func) {
2689 (main_th->unblock.func)(main_th->unblock.arg);
2690 }
2691 }
2692 }
2693 }
2694}
2695
2696#define CLOSE_INVALIDATE_PAIR(expr) \
2697 close_invalidate_pair(expr,"close_invalidate: "#expr)
2698static void
2699close_invalidate(int *fdp, const char *msg)
2700{
2701 int fd = *fdp;
2702
2703 *fdp = -1;
2704 if (close(fd) < 0) {
2705 async_bug_fd(msg, errno, fd);
2706 }
2707}
2708
2709static void
2710close_invalidate_pair(int fds[2], const char *msg)
2711{
2712 if (USE_EVENTFD && fds[0] == fds[1]) {
2713 fds[1] = -1; // disable write port first
2714 close_invalidate(&fds[0], msg);
2715 }
2716 else {
2717 close_invalidate(&fds[1], msg);
2718 close_invalidate(&fds[0], msg);
2719 }
2720}
2721
2722static void
2723set_nonblock(int fd)
2724{
2725 int oflags;
2726 int err;
2727
2728 oflags = fcntl(fd, F_GETFL);
2729 if (oflags == -1)
2730 rb_sys_fail(0);
2731 oflags |= O_NONBLOCK;
2732 err = fcntl(fd, F_SETFL, oflags);
2733 if (err == -1)
2734 rb_sys_fail(0);
2735}
2736
2737/* communication pipe with timer thread and signal handler */
2738static void
2739setup_communication_pipe_internal(int pipes[2])
2740{
2741 int err;
2742
2743 if (pipes[0] > 0 || pipes[1] > 0) {
2744 VM_ASSERT(pipes[0] > 0);
2745 VM_ASSERT(pipes[1] > 0);
2746 return;
2747 }
2748
2749 /*
2750 * Don't bother with eventfd on ancient Linux 2.6.22..2.6.26 which were
2751 * missing EFD_* flags, they can fall back to pipe
2752 */
2753#if USE_EVENTFD && defined(EFD_NONBLOCK) && defined(EFD_CLOEXEC)
2754 pipes[0] = pipes[1] = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
2755
2756 if (pipes[0] >= 0) {
2757 rb_update_max_fd(pipes[0]);
2758 return;
2759 }
2760#endif
2761
2762 err = rb_cloexec_pipe(pipes);
2763 if (err != 0) {
2764 rb_bug("can not create communication pipe");
2765 }
2766 rb_update_max_fd(pipes[0]);
2767 rb_update_max_fd(pipes[1]);
2768 set_nonblock(pipes[0]);
2769 set_nonblock(pipes[1]);
2770}
2771
2772#if !defined(SET_CURRENT_THREAD_NAME) && defined(__linux__) && defined(PR_SET_NAME)
2773# define SET_CURRENT_THREAD_NAME(name) prctl(PR_SET_NAME, name)
2774#endif
2775
2776enum {
2777 THREAD_NAME_MAX =
2778#if defined(__linux__)
2779 16
2780#elif defined(__APPLE__)
2781/* Undocumented, and main thread seems unlimited */
2782 64
2783#else
2784 16
2785#endif
2786};
2787
2788static VALUE threadptr_invoke_proc_location(rb_thread_t *th);
2789
2790static void
2791native_set_thread_name(rb_thread_t *th)
2792{
2793#ifdef SET_CURRENT_THREAD_NAME
2794 VALUE loc;
2795 if (!NIL_P(loc = th->name)) {
2796 SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc));
2797 }
2798 else if ((loc = threadptr_invoke_proc_location(th)) != Qnil) {
2799 char *name, *p;
2800 char buf[THREAD_NAME_MAX];
2801 size_t len;
2802 int n;
2803
2804 name = RSTRING_PTR(RARRAY_AREF(loc, 0));
2805 p = strrchr(name, '/'); /* show only the basename of the path. */
2806 if (p && p[1])
2807 name = p + 1;
2808
2809 n = snprintf(buf, sizeof(buf), "%s:%d", name, NUM2INT(RARRAY_AREF(loc, 1)));
2810 RB_GC_GUARD(loc);
2811
2812 len = (size_t)n;
2813 if (len >= sizeof(buf)) {
2814 buf[sizeof(buf)-2] = '*';
2815 buf[sizeof(buf)-1] = '\0';
2816 }
2817 SET_CURRENT_THREAD_NAME(buf);
2818 }
2819#endif
2820}
2821
2822static void
2823native_set_another_thread_name(rb_nativethread_id_t thread_id, VALUE name)
2824{
2825#if defined SET_ANOTHER_THREAD_NAME || defined SET_CURRENT_THREAD_NAME
2826 char buf[THREAD_NAME_MAX];
2827 const char *s = "";
2828# if !defined SET_ANOTHER_THREAD_NAME
2829 if (!pthread_equal(pthread_self(), thread_id)) return;
2830# endif
2831 if (!NIL_P(name)) {
2832 long n;
2833 RSTRING_GETMEM(name, s, n);
2834 if (n >= (int)sizeof(buf)) {
2835 memcpy(buf, s, sizeof(buf)-1);
2836 buf[sizeof(buf)-1] = '\0';
2837 s = buf;
2838 }
2839 }
2840# if defined SET_ANOTHER_THREAD_NAME
2841 SET_ANOTHER_THREAD_NAME(thread_id, s);
2842# elif defined SET_CURRENT_THREAD_NAME
2843 SET_CURRENT_THREAD_NAME(s);
2844# endif
2845#endif
2846}
2847
2848#if defined(RB_THREAD_T_HAS_NATIVE_ID) || defined(__APPLE__)
2849static VALUE
2850native_thread_native_thread_id(rb_thread_t *target_th)
2851{
2852 if (!target_th->nt) return Qnil;
2853
2854#ifdef RB_THREAD_T_HAS_NATIVE_ID
2855 int tid = target_th->nt->tid;
2856 if (tid == 0) return Qnil;
2857 return INT2FIX(tid);
2858#elif defined(__APPLE__)
2859 uint64_t tid;
2860/* The first condition is needed because MAC_OS_X_VERSION_10_6
2861 is not defined on 10.5, and while __POWERPC__ takes care of ppc/ppc64,
2862 i386 will be broken without this. Note, 10.5 is supported with GCC upstream,
2863 so it has C++17 and everything needed to build modern Ruby. */
2864# if (!defined(MAC_OS_X_VERSION_10_6) || \
2865 (MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_6) || \
2866 defined(__POWERPC__) /* never defined for PowerPC platforms */)
2867 const bool no_pthread_threadid_np = true;
2868# define NO_PTHREAD_MACH_THREAD_NP 1
2869# elif MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_6
2870 const bool no_pthread_threadid_np = false;
2871# else
2872# if !(defined(__has_attribute) && __has_attribute(availability))
2873 /* __API_AVAILABLE macro does nothing on gcc */
2874 __attribute__((weak)) int pthread_threadid_np(pthread_t, uint64_t*);
2875# endif
2876 /* Check weakly linked symbol */
2877 const bool no_pthread_threadid_np = !&pthread_threadid_np;
2878# endif
2879 if (no_pthread_threadid_np) {
2880 return ULL2NUM(pthread_mach_thread_np(pthread_self()));
2881 }
2882# ifndef NO_PTHREAD_MACH_THREAD_NP
2883 int e = pthread_threadid_np(target_th->nt->thread_id, &tid);
2884 if (e != 0) rb_syserr_fail(e, "pthread_threadid_np");
2885 return ULL2NUM((unsigned long long)tid);
2886# endif
2887#endif
2888}
2889# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1
2890#else
2891# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 0
2892#endif
2893
2894static struct {
2895 rb_serial_t created_fork_gen;
2896 pthread_t pthread_id;
2897
2898 int comm_fds[2]; // r, w
2899
2900#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
2901 int event_fd; // kernel event queue fd (epoll/kqueue)
2902#endif
2903#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
2904#define EPOLL_EVENTS_MAX 0x10
2905 struct epoll_event finished_events[EPOLL_EVENTS_MAX];
2906#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
2907#define KQUEUE_EVENTS_MAX 0x10
2908 struct kevent finished_events[KQUEUE_EVENTS_MAX];
2909#endif
2910
2911 // waiting threads list
2912 struct ccan_list_head waiting; // waiting threads in ractors
2913 pthread_mutex_t waiting_lock;
2914} timer_th = {
2915 .created_fork_gen = 0,
2916};
2917
2918#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen)
2919
2920static void timer_thread_check_timeslice(rb_vm_t *vm);
2921static int timer_thread_set_timeout(rb_vm_t *vm);
2922static void timer_thread_wakeup_thread(rb_thread_t *th, uint32_t event_serial);
2923
2924#include "thread_pthread_mn.c"
2925
2926static rb_thread_t *
2927thread_sched_waiting_thread(struct rb_thread_sched_waiting *w)
2928{
2929 if (w) {
2930 return (rb_thread_t *)((size_t)w - offsetof(rb_thread_t, sched.waiting_reason));
2931 }
2932 else {
2933 return NULL;
2934 }
2935}
2936
2937static int
2938timer_thread_set_timeout(rb_vm_t *vm)
2939{
2940#if 0
2941 return 10; // ms
2942#else
2943 int timeout = -1;
2944
2945 ractor_sched_lock(vm, NULL);
2946 {
2947 if ( !ccan_list_empty(&vm->ractor.sched.timeslice_threads) // (1-1) Provide time slice for active NTs
2948 || !ubf_threads_empty() // (1-3) Periodic UBF
2949 || vm->ractor.sched.grq_cnt > 0 // (1-4) Lazy GRQ deq start
2950 ) {
2951
2952 RUBY_DEBUG_LOG("timeslice:%d ubf:%d grq:%d",
2953 !ccan_list_empty(&vm->ractor.sched.timeslice_threads),
2954 !ubf_threads_empty(),
2955 (vm->ractor.sched.grq_cnt > 0));
2956
2957 timeout = 10; // ms
2958 vm->ractor.sched.timeslice_wait_inf = false;
2959 }
2960 else {
2961 vm->ractor.sched.timeslice_wait_inf = true;
2962 }
2963 }
2964 ractor_sched_unlock(vm, NULL);
2965
2966 // Always check waiting threads to find minimum timeout
2967 // even when scheduler has work (grq_cnt > 0)
2968 rb_native_mutex_lock(&timer_th.waiting_lock);
2969 {
2970 struct rb_thread_sched_waiting *w = ccan_list_top(&timer_th.waiting, struct rb_thread_sched_waiting, node);
2971 rb_thread_t *th = thread_sched_waiting_thread(w);
2972
2973 if (th && (th->sched.waiting_reason.flags & thread_sched_waiting_timeout)) {
2974 rb_hrtime_t now = rb_hrtime_now();
2975 rb_hrtime_t hrrel = rb_hrtime_sub(th->sched.waiting_reason.data.timeout, now);
2976
2977 RUBY_DEBUG_LOG("th:%u now:%lu rel:%lu", rb_th_serial(th), (unsigned long)now, (unsigned long)hrrel);
2978
2979 // TODO: overflow?
2980 int thread_timeout = (int)((hrrel + RB_HRTIME_PER_MSEC - 1) / RB_HRTIME_PER_MSEC); // ms
2981
2982 // Use minimum of scheduler timeout and thread sleep timeout
2983 if (timeout < 0 || thread_timeout < timeout) {
2984 timeout = thread_timeout;
2985 }
2986 }
2987 }
2988 rb_native_mutex_unlock(&timer_th.waiting_lock);
2989
2990 RUBY_DEBUG_LOG("timeout:%d inf:%d", timeout, (int)vm->ractor.sched.timeslice_wait_inf);
2991
2992 // fprintf(stderr, "timeout:%d\n", timeout);
2993 return timeout;
2994#endif
2995}
2996
2997static void
2998timer_thread_check_signal(rb_vm_t *vm)
2999{
3000 // ruby_sigchld_handler(vm); TODO
3001
3002 int signum = rb_signal_buff_size();
3003 if (UNLIKELY(signum > 0) && vm->ractor.main_thread) {
3004 RUBY_DEBUG_LOG("signum:%d", signum);
3005 threadptr_trap_interrupt(vm->ractor.main_thread);
3006 }
3007}
3008
3009static bool
3010timer_thread_check_exceed(rb_hrtime_t abs, rb_hrtime_t now)
3011{
3012 if (abs < now) {
3013 return true;
3014 }
3015 else if (abs - now < RB_HRTIME_PER_MSEC) {
3016 return true; // too short time
3017 }
3018 else {
3019 return false;
3020 }
3021}
3022
3023static rb_thread_t *
3024timer_thread_deq_wakeup(rb_vm_t *vm, rb_hrtime_t now, uint32_t *event_serial)
3025{
3026 struct rb_thread_sched_waiting *w = ccan_list_top(&timer_th.waiting, struct rb_thread_sched_waiting, node);
3027
3028 if (w != NULL &&
3029 (w->flags & thread_sched_waiting_timeout) &&
3030 timer_thread_check_exceed(w->data.timeout, now)) {
3031
3032 RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(thread_sched_waiting_thread(w)));
3033
3034 // delete from waiting list
3035 ccan_list_del_init(&w->node);
3036
3037 // setup result
3038 w->flags = thread_sched_waiting_none;
3039 w->data.result = 0;
3040
3041 rb_thread_t *th = thread_sched_waiting_thread(w);
3042 *event_serial = w->data.event_serial;
3043 return th;
3044 }
3045
3046 return NULL;
3047}
3048
3049static void
3050timer_thread_wakeup_thread_locked(struct rb_thread_sched *sched, rb_thread_t *th, uint32_t event_serial)
3051{
3052 if (sched->running != th && th->sched.event_serial == event_serial) {
3053 thread_sched_to_ready_common(sched, th, true, false);
3054 }
3055}
3056
3057static void
3058timer_thread_wakeup_thread(rb_thread_t *th, uint32_t event_serial)
3059{
3060 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
3061 struct rb_thread_sched *sched = TH_SCHED(th);
3062
3063 thread_sched_lock(sched, th);
3064 {
3065 timer_thread_wakeup_thread_locked(sched, th, event_serial);
3066 }
3067 thread_sched_unlock(sched, th);
3068}
3069
3070static void
3071timer_thread_check_timeout(rb_vm_t *vm)
3072{
3073 rb_hrtime_t now = rb_hrtime_now();
3074 rb_thread_t *th;
3075 uint32_t event_serial;
3076
3077 rb_native_mutex_lock(&timer_th.waiting_lock);
3078 {
3079 while ((th = timer_thread_deq_wakeup(vm, now, &event_serial)) != NULL) {
3080 rb_native_mutex_unlock(&timer_th.waiting_lock);
3081 timer_thread_wakeup_thread(th, event_serial);
3082 rb_native_mutex_lock(&timer_th.waiting_lock);
3083 }
3084 }
3085 rb_native_mutex_unlock(&timer_th.waiting_lock);
3086}
3087
3088static void
3089timer_thread_check_timeslice(rb_vm_t *vm)
3090{
3091 // TODO: check time
3092 rb_thread_t *th;
3093 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
3094 RUBY_DEBUG_LOG("timeslice th:%u", rb_th_serial(th));
3095 RUBY_VM_SET_TIMER_INTERRUPT(th->ec);
3096 }
3097}
3098
3099void
3100rb_assert_sig(void)
3101{
3102 sigset_t oldmask;
3103 pthread_sigmask(0, NULL, &oldmask);
3104 if (sigismember(&oldmask, SIGVTALRM)) {
3105 rb_bug("!!!");
3106 }
3107 else {
3108 RUBY_DEBUG_LOG("ok");
3109 }
3110}
3111
3112static void *
3113timer_thread_func(void *ptr)
3114{
3115 rb_vm_t *vm = (rb_vm_t *)ptr;
3116#if defined(RUBY_NT_SERIAL)
3117 ruby_nt_serial = (rb_atomic_t)-1;
3118#endif
3119
3120 RUBY_DEBUG_LOG("started%s", "");
3121
3122 while (RUBY_ATOMIC_LOAD(system_working)) {
3123 timer_thread_check_signal(vm);
3124 timer_thread_check_timeout(vm);
3125 ubf_wakeup_all_threads();
3126
3127 RUBY_DEBUG_LOG("system_working:%d", RUBY_ATOMIC_LOAD(system_working));
3128 timer_thread_polling(vm);
3129 }
3130
3131 RUBY_DEBUG_LOG("terminated");
3132 return NULL;
3133}
3134
3135/* only use signal-safe system calls here */
3136static void
3137signal_communication_pipe(int fd)
3138{
3139#if USE_EVENTFD
3140 const uint64_t buff = 1;
3141#else
3142 const char buff = '!';
3143#endif
3144 ssize_t result;
3145
3146 /* already opened */
3147 if (fd >= 0) {
3148 retry:
3149 if ((result = write(fd, &buff, sizeof(buff))) <= 0) {
3150 int e = errno;
3151 switch (e) {
3152 case EINTR: goto retry;
3153 case EAGAIN:
3154#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
3155 case EWOULDBLOCK:
3156#endif
3157 break;
3158 default:
3159 async_bug_fd("rb_thread_wakeup_timer_thread: write", e, fd);
3160 }
3161 }
3162 if (TT_DEBUG) WRITE_CONST(2, "rb_thread_wakeup_timer_thread: write\n");
3163 }
3164 else {
3165 // ignore wakeup
3166 }
3167}
3168
3169static void
3170timer_thread_wakeup_force(void)
3171{
3172 // should not use RUBY_DEBUG_LOG() because it can be called within signal handlers.
3173 signal_communication_pipe(timer_th.comm_fds[1]);
3174}
3175
3176static void
3177timer_thread_wakeup_locked(rb_vm_t *vm)
3178{
3179 // should be locked before.
3180 ASSERT_ractor_sched_locked(vm, NULL);
3181
3182 if (timer_th.created_fork_gen == current_fork_gen) {
3183 if (vm->ractor.sched.timeslice_wait_inf) {
3184 RUBY_DEBUG_LOG("wakeup with fd:%d", timer_th.comm_fds[1]);
3185 timer_thread_wakeup_force();
3186 }
3187 else {
3188 RUBY_DEBUG_LOG("will be wakeup...");
3189 }
3190 }
3191}
3192
3193static void
3194timer_thread_wakeup(void)
3195{
3196 rb_vm_t *vm = GET_VM();
3197
3198 ractor_sched_lock(vm, NULL);
3199 {
3200 timer_thread_wakeup_locked(vm);
3201 }
3202 ractor_sched_unlock(vm, NULL);
3203}
3204
3205static void
3206rb_thread_create_timer_thread(void)
3207{
3208 rb_serial_t created_fork_gen = timer_th.created_fork_gen;
3209
3210 RUBY_DEBUG_LOG("fork_gen create:%d current:%d", (int)created_fork_gen, (int)current_fork_gen);
3211
3212 timer_th.created_fork_gen = current_fork_gen;
3213
3214 if (created_fork_gen != current_fork_gen) {
3215 if (created_fork_gen != 0) {
3216 RUBY_DEBUG_LOG("forked child process");
3217
3218 CLOSE_INVALIDATE_PAIR(timer_th.comm_fds);
3219#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
3220 close_invalidate(&timer_th.event_fd, "close event_fd");
3221#endif
3222 rb_native_mutex_destroy(&timer_th.waiting_lock);
3223 }
3224
3225 ccan_list_head_init(&timer_th.waiting);
3226 rb_native_mutex_initialize(&timer_th.waiting_lock);
3227
3228 // open communication channel
3229 setup_communication_pipe_internal(timer_th.comm_fds);
3230
3231 // open event fd
3232 timer_thread_setup_mn();
3233 }
3234
3235 pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM());
3236}
3237
3238static int
3239native_stop_timer_thread(void)
3240{
3241 RUBY_ATOMIC_SET(system_working, 0);
3242
3243 RUBY_DEBUG_LOG("wakeup send %d", timer_th.comm_fds[1]);
3244 timer_thread_wakeup_force();
3245 RUBY_DEBUG_LOG("wakeup sent");
3246 pthread_join(timer_th.pthread_id, NULL);
3247
3248 if (TT_DEBUG) fprintf(stderr, "stop timer thread\n");
3249
3250 return 1;
3251}
3252
3253static void
3254native_reset_timer_thread(void)
3255{
3256 //
3257}
3258
3259#ifdef HAVE_SIGALTSTACK
3260int
3261ruby_stack_overflowed_p(const rb_thread_t *th, const void *addr)
3262{
3263 void *base;
3264 size_t size;
3265 const size_t water_mark = 1024 * 1024;
3266 STACK_GROW_DIR_DETECTION;
3267
3268 if (th) {
3269 size = th->ec->machine.stack_maxsize;
3270 base = (char *)th->ec->machine.stack_start - STACK_DIR_UPPER(0, size);
3271 }
3272#ifdef STACKADDR_AVAILABLE
3273 else if (get_stack(&base, &size) == 0) {
3274# ifdef __APPLE__
3275 if (pthread_equal(th->nt->thread_id, native_main_thread.id)) {
3276 struct rlimit rlim;
3277 if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) {
3278 size = (size_t)rlim.rlim_cur;
3279 }
3280 }
3281# endif
3282 base = (char *)base + STACK_DIR_UPPER(+size, -size);
3283 }
3284#endif
3285 else {
3286 return 0;
3287 }
3288
3289 size /= RUBY_STACK_SPACE_RATIO;
3290 if (size > water_mark) size = water_mark;
3291 if (IS_STACK_DIR_UPPER()) {
3292 if (size > ~(size_t)base+1) size = ~(size_t)base+1;
3293 if (addr > base && addr <= (void *)((char *)base + size)) return 1;
3294 }
3295 else {
3296 if (size > (size_t)base) size = (size_t)base;
3297 if (addr > (void *)((char *)base - size) && addr <= base) return 1;
3298 }
3299 return 0;
3300}
3301#endif
3302
3303int
3304rb_reserved_fd_p(int fd)
3305{
3306 /* no false-positive if out-of-FD at startup */
3307 if (fd < 0) return 0;
3308
3309 if (fd == timer_th.comm_fds[0] ||
3310 fd == timer_th.comm_fds[1]
3311#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
3312 || fd == timer_th.event_fd
3313#endif
3314 ) {
3315 goto check_fork_gen;
3316 }
3317 return 0;
3318
3319 check_fork_gen:
3320 if (timer_th.created_fork_gen == current_fork_gen) {
3321 /* async-signal-safe */
3322 return 1;
3323 }
3324 else {
3325 return 0;
3326 }
3327}
3328
3329rb_nativethread_id_t
3331{
3332 return pthread_self();
3333}
3334
3335#if defined(USE_POLL) && !defined(HAVE_PPOLL)
3336/* TODO: don't ignore sigmask */
3337static int
3338ruby_ppoll(struct pollfd *fds, nfds_t nfds,
3339 const struct timespec *ts, const sigset_t *sigmask)
3340{
3341 int timeout_ms;
3342
3343 if (ts) {
3344 int tmp, tmp2;
3345
3346 if (ts->tv_sec > INT_MAX/1000)
3347 timeout_ms = INT_MAX;
3348 else {
3349 tmp = (int)(ts->tv_sec * 1000);
3350 /* round up 1ns to 1ms to avoid excessive wakeups for <1ms sleep */
3351 tmp2 = (int)((ts->tv_nsec + 999999L) / (1000L * 1000L));
3352 if (INT_MAX - tmp < tmp2)
3353 timeout_ms = INT_MAX;
3354 else
3355 timeout_ms = (int)(tmp + tmp2);
3356 }
3357 }
3358 else
3359 timeout_ms = -1;
3360
3361 return poll(fds, nfds, timeout_ms);
3362}
3363# define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask))
3364#endif
3365
3366/*
3367 * Single CPU setups benefit from explicit sched_yield() before ppoll(),
3368 * since threads may be too starved to enter the GVL waitqueue for
3369 * us to detect contention. Instead, we want to kick other threads
3370 * so they can run and possibly prevent us from entering slow paths
3371 * in ppoll() or similar syscalls.
3372 *
3373 * Confirmed on FreeBSD 11.2 and Linux 4.19.
3374 * [ruby-core:90417] [Bug #15398]
3375 */
3376#define THREAD_BLOCKING_YIELD(th) do { \
3377 const rb_thread_t *next_th; \
3378 struct rb_thread_sched *sched = TH_SCHED(th); \
3379 RB_VM_SAVE_MACHINE_CONTEXT(th); \
3380 thread_sched_to_waiting(sched, (th)); \
3381 next_th = sched->running; \
3382 rb_native_mutex_unlock(&sched->lock_); \
3383 native_thread_yield(); /* TODO: needed? */ \
3384 if (!next_th && rb_ractor_living_thread_num(th->ractor) > 1) { \
3385 native_thread_yield(); \
3386 }
3387
3388static void
3389native_sleep(rb_thread_t *th, rb_hrtime_t *rel)
3390{
3391 struct rb_thread_sched *sched = TH_SCHED(th);
3392
3393 RUBY_DEBUG_LOG("rel:%d", rel ? (int)*rel : 0);
3394 if (rel) {
3395 if (th_has_dedicated_nt(th)) {
3396 native_cond_sleep(th, rel);
3397 }
3398 else {
3399 thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel);
3400 }
3401 }
3402 else {
3403 thread_sched_to_waiting_until_wakeup(sched, th);
3404 }
3405
3406 RUBY_DEBUG_LOG("wakeup");
3407}
3408
3409// fork read-write lock (only for pthread)
3410static pthread_rwlock_t rb_thread_fork_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3411
3412// the registered thread waiting to be joined, or NULL; no pthread_t is 0
3413static void *last_exiting_thread;
3414STATIC_ASSERT(pthread_t_fits_in_pointer, sizeof(pthread_t) <= sizeof(void *));
3415
3416void
3417rb_thread_release_fork_lock(void)
3418{
3419 int r;
3420 if ((r = pthread_rwlock_unlock(&rb_thread_fork_rw_lock))) {
3421 rb_bug_errno("pthread_rwlock_unlock", r);
3422 }
3423}
3424
3425void
3426rb_thread_reset_fork_lock(void)
3427{
3428 // On macOS the child's copy still counts a thread waiting for this lock
3429 // at fork: unlocking it leaves kernel state that later waiters hang on,
3430 // and destroying or initializing it fails with EBUSY.
3431 rb_thread_fork_rw_lock = (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
3432}
3433
3434void *
3435rb_thread_prevent_fork(void *(*func)(void *), void *data)
3436{
3437 int r;
3438 if ((r = pthread_rwlock_rdlock(&rb_thread_fork_rw_lock))) {
3439 rb_bug_errno("pthread_rwlock_rdlock", r);
3440 }
3441 void *result = func(data);
3442 rb_thread_release_fork_lock();
3443 return result;
3444}
3445
3446static void
3447thread_register_exiting(void *thread)
3448{
3449 void *previous_thread = RUBY_ATOMIC_PTR_EXCHANGE(last_exiting_thread, thread);
3450 if (previous_thread) {
3451 int r;
3452 if ((r = pthread_join((pthread_t)(uintptr_t)previous_thread, NULL))) {
3453 rb_bug_errno("pthread_join", r);
3454 }
3455 }
3456}
3457
3458// Called within rb_thread_prevent_fork() by a joinable thread just before it
3459// exits. The thread it replaces in the slot has usually exited by then.
3460void
3461rb_thread_register_exiting(void)
3462{
3463 thread_register_exiting((void *)(uintptr_t)pthread_self());
3464}
3465
3466void
3467rb_thread_acquire_fork_lock(void)
3468{
3469 int r;
3470 if ((r = pthread_rwlock_wrlock(&rb_thread_fork_rw_lock))) {
3471 rb_bug_errno("pthread_rwlock_wrlock", r);
3472 }
3473
3474 // A registered thread may still be exiting, and glibc's thread exit takes
3475 // the resolver configuration lock, which fork does not reset. None can
3476 // register while this lock is held.
3477 thread_register_exiting(NULL);
3478}
3479
3480// thread internal event hooks (only for pthread)
3481
3482struct rb_internal_thread_event_hook {
3483 rb_internal_thread_event_callback callback;
3484 rb_event_flag_t event;
3485 void *user_data;
3486
3487 struct rb_internal_thread_event_hook *next;
3488};
3489
3490static pthread_rwlock_t rb_internal_thread_event_hooks_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3491
3492#if defined(HAVE_WORKING_FORK)
3493void
3494rb_internal_thread_event_hooks_rw_lock_atfork(void)
3495{
3496 // After fork(), this rwlock may have been held by a now-dead thread.
3497 //
3498 // pthread_rwlock_destroy() on a held lock is undefined behavior, and
3499 // pthread_rwlock_init() on an already-initialized lock is also undefined
3500 // behavior
3501 //
3502 // Direct assignment of PTHREAD_RWLOCK_INITIALIZER is safe and portable.
3503 rb_internal_thread_event_hooks_rw_lock =
3504 (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
3505}
3506#endif
3507
3508rb_internal_thread_event_hook_t *
3509rb_internal_thread_add_event_hook(rb_internal_thread_event_callback callback, rb_event_flag_t internal_event, void *user_data)
3510{
3511 rb_internal_thread_event_hook_t *hook = ALLOC_N(rb_internal_thread_event_hook_t, 1);
3512 hook->callback = callback;
3513 hook->user_data = user_data;
3514 hook->event = internal_event;
3515
3516 int r;
3517 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3518 rb_bug_errno("pthread_rwlock_wrlock", r);
3519 }
3520
3521 hook->next = rb_internal_thread_event_hooks;
3522 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook);
3523
3524 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3525 rb_bug_errno("pthread_rwlock_unlock", r);
3526 }
3527 return hook;
3528}
3529
3530bool
3531rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t * hook)
3532{
3533 int r;
3534 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3535 rb_bug_errno("pthread_rwlock_wrlock", r);
3536 }
3537
3538 bool success = FALSE;
3539
3540 if (rb_internal_thread_event_hooks == hook) {
3541 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook->next);
3542 success = TRUE;
3543 }
3544 else {
3545 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3546
3547 do {
3548 if (h->next == hook) {
3549 h->next = hook->next;
3550 success = TRUE;
3551 break;
3552 }
3553 } while ((h = h->next));
3554 }
3555
3556 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3557 rb_bug_errno("pthread_rwlock_unlock", r);
3558 }
3559
3560 if (success) {
3561 ruby_xfree(hook);
3562 }
3563 return success;
3564}
3565
3566static void
3567rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th)
3568{
3569 int r;
3570 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
3571 rb_bug_errno("pthread_rwlock_rdlock", r);
3572 }
3573
3574 if (rb_internal_thread_event_hooks) {
3575 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3576 do {
3577 if (h->event & event) {
3578 rb_internal_thread_event_data_t event_data = {
3579 .thread = th->self,
3580 };
3581 (*h->callback)(event, &event_data, h->user_data);
3582 }
3583 } while((h = h->next));
3584 }
3585 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3586 rb_bug_errno("pthread_rwlock_unlock", r);
3587 }
3588}
3589
3590// return true if the current thread acquires DNT.
3591// return false if the current thread already acquires DNT.
3592bool
3594{
3595 rb_thread_t *th = GET_THREAD();
3596 bool is_snt = th->nt->dedicated == 0;
3597 native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
3598
3599 return is_snt;
3600}
3601
3602void
3603rb_thread_malloc_stack_set(rb_thread_t *th, void *stack)
3604{
3605 th->sched.malloc_stack = true;
3606 th->sched.context_stack = stack;
3607}
3608
3609#endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
Definition atomic.h:69
#define RUBY_ATOMIC_PTR_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are void*.
Definition atomic.h:327
#define RUBY_ATOMIC_FETCH_ADD(var, val)
Atomically replaces the value pointed by var with the result of addition of val to the old value of v...
Definition atomic.h:118
#define RUBY_ATOMIC_LOAD(var)
Atomic load.
Definition atomic.h:175
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
Definition atomic.h:185
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
#define ZALLOC
Old name of RB_ZALLOC.
Definition memory.h:402
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:399
#define ULL2NUM
Old name of RB_ULL2NUM.
Definition long_long.h:31
#define NUM2INT
Old name of RB_NUM2INT.
Definition int.h:44
#define Qnil
Old name of RUBY_Qnil.
#define NIL_P
Old name of RB_NIL_P.
VALUE rb_eNotImpError
NotImplementedError exception.
Definition error.c:1441
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
Definition error.c:3911
void rb_bug_errno(const char *mesg, int errno_arg)
This is a wrapper of rb_bug() which automatically constructs appropriate message from the passed errn...
Definition error.c:1141
int rb_cloexec_pipe(int fildes[2])
Opens a pipe with closing on exec.
Definition io.c:514
void rb_update_max_fd(int fd)
Informs the interpreter that the passed fd can be the max.
Definition io.c:286
int rb_reserved_fd_p(int fd)
Queries if the given FD is reserved or not.
void rb_unblock_function_t(void *)
This is the type of UBFs.
Definition thread.h:336
void rb_timespec_now(struct timespec *ts)
Fills the current time into the given struct.
Definition time.c:2003
char * ptr
Pointer to the underlying memory region, of at least capa bytes.
Definition io.h:2
int len
Length of the buffer.
Definition io.h:8
#define RUBY_INTERNAL_THREAD_EVENT_RESUMED
Triggered when a thread successfully acquired the GVL.
Definition thread.h:238
rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback func, rb_event_flag_t events, void *data)
Registers a thread event hook function.
#define RUBY_INTERNAL_THREAD_EVENT_EXITED
Triggered when a thread exits.
Definition thread.h:252
#define RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
Triggered when a thread released the GVL.
Definition thread.h:245
bool rb_thread_lock_native_thread(void)
Declare the current Ruby thread should acquire a dedicated native thread on M:N thread scheduler.
#define RUBY_INTERNAL_THREAD_EVENT_STARTED
Triggered when a new thread is started.
Definition thread.h:224
bool rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t *hook)
Unregister the passed hook.
#define RUBY_INTERNAL_THREAD_EVENT_READY
Triggered when a thread attempt to acquire the GVL.
Definition thread.h:231
#define RBIMPL_ATTR_MAYBE_UNUSED()
Wraps (or simulates) [[maybe_unused]].
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
#define rb_fd_select
Waits for multiple file descriptors at once.
Definition posix.h:66
#define RARRAY_AREF(a, i)
Definition rarray.h:403
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
Definition rstring.h:450
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
The data structure which wraps the fd_set bitmap used by select(2).
Definition largesize.h:71
Definition string.c:8285
rb_nativethread_id_t rb_nativethread_self(void)
Queries the ID of the native thread that is calling this function.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
int rb_native_mutex_trylock(rb_nativethread_lock_t *lock)
Identical to rb_native_mutex_lock(), except it doesn't block in case rb_native_mutex_lock() would.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
void rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec)
Identical to rb_native_cond_wait(), except it additionally takes timeout in msec resolution.
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
void ruby_xfree(void *ptr)
Deallocates a storage instance.
Definition gc.c:5283