diff --git a/.gitignore b/.gitignore index 950e8cc3..fdfe9ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ /benchmarks/discourse .DS_Store +/benchmarks/ractor-conservative-stack/build diff --git a/benchmarks.yml b/benchmarks.yml index d11b7570..ed38422e 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -296,6 +296,11 @@ json_parse_string: ractor: true ractor_only: true default_harness: harness-ractor +ractor-conservative-stack: + desc: ractors each build a deep Ruby -> C -> Ruby call stack with large C frames and run minor GCs at the bottom, to stress conservative machine stack scanning. + ractor: true + ractor_only: true + default_harness: harness-ractor ractor-dead-set: desc: multiple ractors each build a large live set and terminate, then memory retention after their death is measured. ractor: true diff --git a/benchmarks/ractor-conservative-stack/benchmark.rb b/benchmarks/ractor-conservative-stack/benchmark.rb new file mode 100644 index 00000000..a3728f53 --- /dev/null +++ b/benchmarks/ractor-conservative-stack/benchmark.rb @@ -0,0 +1,125 @@ +# Every worker builds a deep, production-like machine stack by bouncing +# Ruby -> C -> Ruby -> C through a C extension whose frames hold many stack +# slots: live VALUEs, interior pointers, C pointers and scalars. At the bottom +# of the stack it allocates garbage and runs a minor GC, so every collection +# scans the whole stack conservatively (gc_mark_maybe). Objects referenced +# only from C stack slots are checked when each frame returns; a missed +# conservative root raises RactorStackScan::CorruptionError. +# +# Each worker does the same work at every Ractor count. + +Warning[:experimental] = false + +require_relative "../../harness/loader" +require "digest" +require "fileutils" +require "shellwords" + +def env_int(name, default, range = 0..) + value = Integer(ENV.fetch(name, default)) + raise ArgumentError, "#{name} must be in #{range}" unless range.cover?(value) + value +end + +DEPTH = env_int("RACTOR_CONSERVATIVE_STACK_DEPTH", 48) +FRAME_SLOTS = env_int("RACTOR_CONSERVATIVE_STACK_FRAME_SLOTS", 256, 1..) +FRAME_SCALARS = env_int("RACTOR_CONSERVATIVE_STACK_FRAME_SCALARS", 256) +DESCENTS = env_int("RACTOR_CONSERVATIVE_STACK_DESCENTS", 40, 1..) +LEAF_ALLOCS = env_int("RACTOR_CONSERVATIVE_STACK_LEAF_ALLOCS", 100_000) +POOL_SIZE = env_int("RACTOR_CONSERVATIVE_STACK_POOL_SIZE", 20_000, 1..) + +def build_extension + ext_dir = File.join(__dir__, "ext") + sources = %w[ractor_stack_scan.c extconf.rb].map { |f| File.read(File.join(ext_dir, f)) } + flags = RbConfig::CONFIG.values_at("CC", "CFLAGS", "optflags", "debugflags", "configure_args") + key = Digest::SHA256.hexdigest([*sources, RUBY_PLATFORM, RUBY_REVISION, RbConfig.ruby, *flags].join("\0"))[0, 16] + build_dir = File.join(__dir__, "build", key) + lib = File.join(build_dir, "ractor_stack_scan.#{RbConfig::CONFIG["DLEXT"]}") + + FileUtils.mkdir_p(build_dir) + Dir.chdir(build_dir) do + setup_cmds([[RbConfig.ruby, File.join(ext_dir, "extconf.rb")].shelljoin]) unless File.exist?("Makefile") + setup_cmds([ENV.fetch("MAKE", "make")]) + end + lib +end + +require build_extension + +max_slots = RactorStackScan::MAX_FRAME_SLOTS +slot_kinds = RactorStackScan::SLOT_KINDS +if FRAME_SLOTS > max_slots || FRAME_SLOTS % slot_kinds != 0 + raise ArgumentError, "RACTOR_CONSERVATIVE_STACK_FRAME_SLOTS must be a multiple of #{slot_kinds} up to #{max_slots}" +end +if FRAME_SCALARS > max_slots + raise ArgumentError, "RACTOR_CONSERVATIVE_STACK_FRAME_SCALARS must be <= #{max_slots}" +end +if DEPTH > RactorStackScan::MAX_DEPTH + raise ArgumentError, "RACTOR_CONSERVATIVE_STACK_DEPTH must be <= #{RactorStackScan::MAX_DEPTH}" +end + +# A native stack overflow in a worker Ractor crashes the process instead of +# raising SystemStackError. Each level also needs about 3 KiB for the VM frames +# of the Ruby -> C round trip, measured on arm64 macOS. +word = RactorStackScan::WORD_SIZE +frame_bytes = (FRAME_SLOTS + FRAME_SCALARS + 1) * word + RactorStackScan::MAX_OWNED_STRING_LEN +vm_bytes_per_level = 3 * 1024 +estimated_stack = (DEPTH + 1) * (frame_bytes + vm_bytes_per_level) +thread_stack = [ENV["RUBY_THREAD_MACHINE_STACK_SIZE"]&.to_i || 128 * 1024 * word, 16 * 1024 * word].max +if estimated_stack > thread_stack * 9 / 10 + raise ArgumentError, "estimated machine stack #{estimated_stack / 1024} KiB exceeds 90% of the " \ + "Ractor thread stack (#{thread_stack / 1024} KiB); lower the depth or frame size, " \ + "or raise RUBY_THREAD_MACHINE_STACK_SIZE" +end + +GC_START_KWARGS = { full_mark: false, immediate_sweep: true } +GC_START_KWARGS[:global] = false if GC.method(:start).parameters.include?([:key, :global]) +Ractor.make_shareable(GC_START_KWARGS) + +class StackWalker + def initialize + @pool = Array.new(POOL_SIZE) do |i| + case i % 6 + when 0 then "pooled string #{i}" + when 1 then "long pooled string #{i} " * 32 + when 2 then [i, i.to_s, :pooled] + when 3 then { key: i, value: i.to_s } + when 4 then Object.new + else (2**70) + i + end + end + @retained = Array.new([POOL_SIZE / 4, 1].max) + @retain_cursor = 0 + end + + def run + DESCENTS.times { descend(DEPTH) } + end + + private + + def descend(depth) + return leaf if depth.negative? + + RactorStackScan.native_frame(self, depth, @pool, FRAME_SLOTS, FRAME_SCALARS) { |next_depth| descend(next_depth) } + end + + def leaf + i = 0 + rounds = LEAF_ALLOCS / 4 + while i < rounds + str = "churn #{i}" + hash = { [str, i] => str } + if i % 16 == 0 + @retained[@retain_cursor] = hash + @retain_cursor = (@retain_cursor + 1) % @retained.size + end + i += 1 + end + GC.start(**GC_START_KWARGS) + end +end + +run_benchmark(5) do |_num_ractors = 0| + StackWalker.new.run +end diff --git a/benchmarks/ractor-conservative-stack/ext/extconf.rb b/benchmarks/ractor-conservative-stack/ext/extconf.rb new file mode 100644 index 00000000..4b2ac964 --- /dev/null +++ b/benchmarks/ractor-conservative-stack/ext/extconf.rb @@ -0,0 +1,2 @@ +require "mkmf" +create_makefile("ractor_stack_scan") diff --git a/benchmarks/ractor-conservative-stack/ext/ractor_stack_scan.c b/benchmarks/ractor-conservative-stack/ext/ractor_stack_scan.c new file mode 100644 index 00000000..01917106 --- /dev/null +++ b/benchmarks/ractor-conservative-stack/ext/ractor_stack_scan.c @@ -0,0 +1,220 @@ +#include +#include +#include +#include + +#define MAX_FRAME_SLOTS 16384 +#define SLOT_KINDS 8 +#define MAX_OWNED_STRING_LEN 1024 +#define MAX_DEPTH (LONG_MAX / MAX_FRAME_SLOTS - 1) + +static ID id_descend; +static VALUE eCorruptionError; + +struct descend_args { + VALUE walker; + VALUE depth; +}; + +static VALUE +descend_protected(VALUE ptr) +{ + struct descend_args *args = (struct descend_args *)ptr; + return rb_funcall(args->walker, id_descend, 1, args->depth); +} + +static long +owned_string_len(long slot) +{ + if (slot % (SLOT_KINDS * 16) == 0) return MAX_OWNED_STRING_LEN; + if (slot % (SLOT_KINDS * 2) == 0) return 96; + return 16; +} + +static void +owned_pattern(char *buf, long len, uintptr_t nonce, long depth, long slot) +{ + uint64_t x = (uint64_t)nonce ^ ((uint64_t)depth << 32) ^ (uint64_t)slot; + for (long i = 0; i < len; i++) { + x = x * 6364136223846793005ULL + 1442695040888963407ULL; + buf[i] = (char)('a' + (char)((x >> 59) % 26)); + } +} + +static void +verify_owned_string(VALUE str, uintptr_t nonce, long depth, long slot) +{ + char expected[MAX_OWNED_STRING_LEN]; + long len = owned_string_len(slot); + + if (!RB_TYPE_P(str, T_STRING) || RSTRING_LEN(str) != len) { + rb_raise(eCorruptionError, "string held only by a C stack slot was collected (depth %ld, slot %ld)", depth, slot); + } + owned_pattern(expected, len, nonce, depth, slot); + if (memcmp(RSTRING_PTR(str), expected, (size_t)len) != 0) { + rb_raise(eCorruptionError, "string held only by a C stack slot was overwritten (depth %ld, slot %ld)", depth, slot); + } +} + +static long +owned_array_tag(uintptr_t nonce, long depth) +{ + return depth ^ (long)(nonce >> 3); +} + +static void +verify_owned_array(VALUE ary, uintptr_t nonce, long depth, long slot) +{ + if (!RB_TYPE_P(ary, T_ARRAY) || RARRAY_LEN(ary) != 3 || + RARRAY_AREF(ary, 0) != LONG2FIX(slot) || RARRAY_AREF(ary, 2) != LONG2FIX(owned_array_tag(nonce, depth))) { + rb_raise(eCorruptionError, "array held only by a C stack slot was collected or overwritten (depth %ld, slot %ld)", depth, slot); + } +} + +/* + * One native frame of the descent. The volatile arrays keep every slot in + * memory, so a GC deeper in the call chain must find the owned strings and + * arrays by scanning this frame conservatively. + */ +static VALUE +native_frame(VALUE self, VALUE walker, VALUE vdepth, VALUE pool, VALUE vslots, VALUE vscalars) +{ + long depth = NUM2LONG(vdepth); + long nslots = NUM2LONG(vslots); + long nscalars = NUM2LONG(vscalars); + long pool_len; + VALUE result; + + Check_Type(pool, T_ARRAY); + pool_len = RARRAY_LEN(pool); + if (pool_len == 0) rb_raise(rb_eArgError, "pool must not be empty"); + if (depth < 0 || depth > MAX_DEPTH) rb_raise(rb_eArgError, "depth must be in 0..%ld", (long)MAX_DEPTH); + if (nslots < SLOT_KINDS || nslots > MAX_FRAME_SLOTS || nslots % SLOT_KINDS != 0) { + rb_raise(rb_eArgError, "frame slots must be a multiple of %d in %d..%d", SLOT_KINDS, SLOT_KINDS, MAX_FRAME_SLOTS); + } + if (nscalars < 0 || nscalars > MAX_FRAME_SLOTS) { + rb_raise(rb_eArgError, "frame scalars must be in 0..%d", MAX_FRAME_SLOTS); + } + + volatile VALUE slots[nslots]; + volatile uintptr_t scalars[nscalars + 1]; + char pattern[MAX_OWNED_STRING_LEN]; + /* + * The frame address makes the expected string bytes and array tag specific + * to this native frame, so an object allocated into a missed root's freed slot is + * unlikely to pass verification. Depth and slot alone repeat in every + * Ractor that runs the same descent. + */ + uintptr_t nonce = (uintptr_t)&slots[0]; + + for (long i = 0; i < nslots; i++) slots[i] = Qnil; + + /* + * Only kinds 0 (owned string) and 4 (owned array) are roots: nothing else + * references them, and they are verified after the call returns. The other + * kinds, and the scalars below, are the words a real C frame also holds: + * immediates, integers, stack addresses and interior pointers. The + * conservative scan must reject or harmlessly mark each of them, and they + * take different paths through it. + */ + for (long i = 0; i < nslots; i++) { + VALUE pooled = RARRAY_AREF(pool, (depth * nslots + i) % pool_len); + + switch (i % SLOT_KINDS) { + case 0: { + long len = owned_string_len(i); + owned_pattern(pattern, len, nonce, depth, i); + slots[i] = rb_str_new(pattern, len); + break; + } + case 1: + case 5: + slots[i] = pooled; + break; + case 2: + slots[i] = pooled + sizeof(VALUE); + break; + case 3: + slots[i] = (VALUE)RSTRING_PTR(slots[i - 3]); + break; + case 4: + slots[i] = rb_ary_new_from_args(3, LONG2FIX(i), pooled, LONG2FIX(owned_array_tag(nonce, depth))); + break; + case 6: + slots[i] = (i & SLOT_KINDS) ? LONG2FIX(i) : ID2SYM(id_descend); + break; + default: + slots[i] = (VALUE)&slots[i - 7]; + break; + } + } + + for (long j = 0; j < nscalars; j++) { + switch (j % 4) { + case 0: + scalars[j] = (uintptr_t)(j * (depth + 1)); + break; + case 1: { + double d = (double)j / (double)(depth + 1); + uint64_t bits; + memcpy(&bits, &d, sizeof(bits)); + scalars[j] = (uintptr_t)bits; + break; + } + case 2: + scalars[j] = (uintptr_t)RSTRING_PTR(slots[(j % (nslots / SLOT_KINDS)) * SLOT_KINDS]); + break; + default: + scalars[j] = (uintptr_t)&scalars[j]; + break; + } + } + scalars[nscalars] = (uintptr_t)pattern; + + VALUE next = LONG2NUM(depth - 1); + + /* + * Enter the next level three ways, so the scanned stack also holds + * rb_protect's jmp_buf (saved registers that may hold VALUEs) and the VM + * frames of a block call, as real extensions produce. + */ + switch (depth % 3) { + case 0: + result = rb_funcall(walker, id_descend, 1, next); + break; + case 1: { + struct descend_args args = { walker, next }; + int state = 0; + result = rb_protect(descend_protected, (VALUE)&args, &state); + if (state) rb_jump_tag(state); + break; + } + default: + result = rb_block_given_p() ? rb_yield(next) : rb_funcall(walker, id_descend, 1, next); + break; + } + + for (long i = 0; i < nslots; i += SLOT_KINDS) { + verify_owned_string(slots[i], nonce, depth, i); + verify_owned_array(slots[i + 4], nonce, depth, i + 4); + } + + return result; +} + +void +Init_ractor_stack_scan(void) +{ +#ifdef HAVE_RB_EXT_RACTOR_SAFE + rb_ext_ractor_safe(true); +#endif + VALUE mod = rb_define_module("RactorStackScan"); + eCorruptionError = rb_define_class_under(mod, "CorruptionError", rb_eRuntimeError); + id_descend = rb_intern("descend"); + rb_define_const(mod, "MAX_FRAME_SLOTS", INT2FIX(MAX_FRAME_SLOTS)); + rb_define_const(mod, "MAX_DEPTH", LONG2NUM(MAX_DEPTH)); + rb_define_const(mod, "SLOT_KINDS", INT2FIX(SLOT_KINDS)); + rb_define_const(mod, "MAX_OWNED_STRING_LEN", INT2FIX(MAX_OWNED_STRING_LEN)); + rb_define_const(mod, "WORD_SIZE", INT2FIX(sizeof(VALUE))); + rb_define_module_function(mod, "native_frame", native_frame, 5); +}