From 7e7abff0cb5bc6e352dddc4d5b9a4beaa45c9bce Mon Sep 17 00:00:00 2001 From: Sam Ruby Date: Sat, 10 Oct 2026 07:56:48 -0400 Subject: [PATCH 1/4] Ractor support Declare the C extension Ractor-safe when SQLite is built thread-safe, so that each Ractor can open and use its own connections. This finishes #365, rebased onto main, with these changes: - Fill the statement-status symbol table at Init. It was filled lazily on first use, and the "already filled?" check only looked at slot 0, so a second Ractor could see a partly filled table. - define_aggregator: capture the template and name in closures instead of class instance variables, which can't be set outside the main Ractor (Ractor::IsolationError). - Make VERSION_INFO deeply frozen so any Ractor can read it. - Drop the FULLMUTEX "shareable database" part. A Database also holds Ruby state (registered functions, aggregators, collations, handlers), which can't be shared between Ractors even when SQLite's calls are serialized. - Run the Ractor tests in a child process with a timeout. Once a process has started a Ractor, ObjectSpace.each_object stops seeing objects (Ruby 4.0), which broke test_aggregate_instances_are_released_after_each_query when the Ractor tests ran first. Co-authored-by: Roman Sandler <5535625+sandlerr@users.noreply.github.com> Co-authored-by: Aaron Patterson Co-authored-by: Mike Dalessio Co-Authored-By: Claude Opus 5.5 --- RACTOR.md | 85 +++++++++++++++++++++ ext/sqlite3/extconf.rb | 3 + ext/sqlite3/sqlite3.c | 6 ++ ext/sqlite3/statement.c | 23 +++--- lib/sqlite3.rb | 5 ++ lib/sqlite3/database.rb | 23 ++---- lib/sqlite3/version_info.rb | 6 +- test/test_integration_ractor.rb | 129 ++++++++++++++++++++++++++++++++ 8 files changed, 251 insertions(+), 29 deletions(-) create mode 100644 RACTOR.md create mode 100644 test/test_integration_ractor.rb diff --git a/RACTOR.md b/RACTOR.md new file mode 100644 index 00000000..0d8ef248 --- /dev/null +++ b/RACTOR.md @@ -0,0 +1,85 @@ +# Ractor support in SQLite3-Ruby + +SQLite3 has [3 different modes of threading support](https://www.sqlite.org/threadsafe.html). + +1. Single-thread +2. Multi-thread +3. Serialized + +"Single thread" mode means there are no mutexes, so the library itself is not +thread safe. In other words if two threads do `SQLite3::Database.new` on the +same file, it will have thread safety problems. + +"Multi thread" mode means that SQLite3 puts mutexes in place, but it does not +mean that the SQLite3 API itself is thread safe. In other words, in this mode +it is SAFE for two threads to do `SQLite3::Database.new` on the same file, but +it is NOT SAFE to share that database object between two threads. + +"Serialized" mode is like "Multi thread" mode except that there are mutexes in +place such that it IS SAFE to share the same database object between threads. + +## Ractor Safety + +When a C extension claims to be Ractor safe by calling `rb_ext_ractor_safe`, +it's merely claiming that its C API is thread safe. This _does not_ mean that +objects allocated from said C extension are allowed to cross between Ractor +boundaries. + +In other words, `rb_ext_ractor_safe` matches the expectations of the +"multi-thread" mode of SQLite3. We can detect the multithread mode via the +`sqlite3_threadsafe` function. In other words, it's fine to declare this +extension is Ractor safe, but only if `sqlite3_threadsafe` returns true. +`SQLite3.ractor_safe?` reports whether the extension made that declaration. + +The supported pattern is that each Ractor opens and uses its own connections. +Everything works inside a Ractor, including user-defined functions, +aggregators, collations, the authorizer, `trace`, and `busy_handler`. Several +Ractors may open connections to the same database file; SQLite's own locking +coordinates them, just as it does for separate processes. + +```ruby +require "sqlite3" + +ractors = 4.times.map do |i| + Ractor.new(i) do |i| + db = SQLite3::Database.new("app.db") + db.busy_timeout = 5000 + db.execute("INSERT INTO log (worker) VALUES (?)", [i]) + db.close + end +end +ractors.each(&:value) +``` + +## Database objects can't be passed between Ractors + +A `SQLite3::Database` belongs to the Ractor that opened it. Passing one to +another Ractor raises a `Ractor::Error`: + +```ruby +require "sqlite3" + +r = Ractor.new { Ractor.receive } + +db = SQLite3::Database.new ":memory:" + +begin + r.send db + puts "unreachable" +rescue Ractor::Error +end +``` + +This is true even for a connection opened in "Serialized" mode +(`SQLite3::Constants::Open::FULLMUTEX`). Serialized mode makes the SQLite +calls safe, but a `Database` also holds Ruby state: the procs and objects +registered as functions, aggregators, collations, and handlers. Those can't +be shared between Ractors. + +## Fork Safety + +Fork safety is restricted to database objects that were created on the main +Ractor. When a process forks, the child process shuts down all Ractors, so +any database connections that are inside a Ractor should be released. +Because a connection can't be passed between Ractors, every connection the main +Ractor can reach is tracked. diff --git a/ext/sqlite3/extconf.rb b/ext/sqlite3/extconf.rb index 1fbefd73..8fd4e9f3 100644 --- a/ext/sqlite3/extconf.rb +++ b/ext/sqlite3/extconf.rb @@ -129,6 +129,9 @@ def configure_extension # Functions defined in 2.1 but not 2.0 have_func("rb_integer_pack") + # Functions defined in 3.0 but not 2.7 + have_func("rb_ext_ractor_safe") + # These functions may not be defined have_func("sqlite3_initialize") have_func("sqlite3_backup_init") diff --git a/ext/sqlite3/sqlite3.c b/ext/sqlite3/sqlite3.c index d30e85d5..06e439e9 100644 --- a/ext/sqlite3/sqlite3.c +++ b/ext/sqlite3/sqlite3.c @@ -127,6 +127,12 @@ init_sqlite3_constants(void) VALUE mSqlite3Constants; VALUE mSqlite3Open; +#ifdef HAVE_RB_EXT_RACTOR_SAFE + if (sqlite3_threadsafe()) { + rb_ext_ractor_safe(true); + } +#endif + mSqlite3Constants = rb_define_module_under(mSqlite3, "Constants"); /* sqlite3_open_v2 flags for Database::new */ diff --git a/ext/sqlite3/statement.c b/ext/sqlite3/statement.c index 9d0c416b..fac984ea 100644 --- a/ext/sqlite3/statement.c +++ b/ext/sqlite3/statement.c @@ -584,26 +584,24 @@ static VALUE stmt_stat_symbols[stmt_stat_sym_last]; static void setup_stmt_stat_symbols(void) { - if (stmt_stat_symbols[0] == 0) { #define S(s) stmt_stat_symbols[stmt_stat_sym_##s] = ID2SYM(rb_intern_const(#s)) - S(fullscan_steps); - S(sorts); - S(autoindexes); - S(vm_steps); + S(fullscan_steps); + S(sorts); + S(autoindexes); + S(vm_steps); #ifdef SQLITE_STMTSTATUS_REPREPARE - S(reprepares); + S(reprepares); #endif #ifdef SQLITE_STMTSTATUS_RUN - S(runs); + S(runs); #endif #ifdef SQLITE_STMTSTATUS_FILTER_MISS - S(filter_misses); + S(filter_misses); #endif #ifdef SQLITE_STMTSTATUS_FILTER_HIT - S(filter_hits); + S(filter_hits); #endif #undef S - } } static size_t @@ -611,8 +609,6 @@ stmt_stat_internal(VALUE hash_or_sym, sqlite3_stmt *stmt) { VALUE hash = Qnil, key = Qnil; - setup_stmt_stat_symbols(); - if (RB_TYPE_P(hash_or_sym, T_HASH)) { hash = hash_or_sym; } else if (SYMBOL_P(hash_or_sym)) { @@ -776,6 +772,9 @@ init_sqlite3_statement(void) { cSqlite3Statement = rb_define_class_under(mSqlite3, "Statement", rb_cObject); + /* populate eagerly: a lazy fill would race when the extension is used from multiple Ractors */ + setup_stmt_stat_symbols(); + rb_define_alloc_func(cSqlite3Statement, allocate); rb_define_method(cSqlite3Statement, "close", sqlite3_rb_close, 0); rb_define_method(cSqlite3Statement, "closed?", closed_p, 0); diff --git a/lib/sqlite3.rb b/lib/sqlite3.rb index 93caef14..837c936f 100644 --- a/lib/sqlite3.rb +++ b/lib/sqlite3.rb @@ -14,6 +14,11 @@ module SQLite3 def self.threadsafe? threadsafe > 0 end + + # Is the gem's C extension marked as Ractor-safe? + def self.ractor_safe? + threadsafe? && !defined?(Ractor).nil? + end end require "sqlite3/version_info" diff --git a/lib/sqlite3/database.rb b/lib/sqlite3/database.rb index 6fbff7b0..7ddf5134 100644 --- a/lib/sqlite3/database.rb +++ b/lib/sqlite3/database.rb @@ -180,7 +180,7 @@ def initialize file, options = {}, zvfs = nil initialize_extensions(options[:extensions]) - ForkSafety.track(self) + ForkSafety.track(self) if !defined?(Ractor) || (Ractor.main == Ractor.current) if block_given? begin @@ -592,22 +592,15 @@ def finalize def define_aggregator(name, aggregator) # Previously, this has been implemented in C. Now this is just yet # another compatibility shim + # + # The template and name are captured in closures rather than class + # instance variables, which can't be set outside the main Ractor. proxy = Class.new do - @template = aggregator - @name = name - - def self.template - @template - end + define_singleton_method(:template) { aggregator } + define_singleton_method(:name) { name } - def self.name - @name - end - - def self.arity - # this is what sqlite3_obj_method_arity did before - @template.method(:step).arity - end + # this is what sqlite3_obj_method_arity did before + define_singleton_method(:arity) { aggregator.method(:step).arity } def initialize @klass = self.class.template.clone diff --git a/lib/sqlite3/version_info.rb b/lib/sqlite3/version_info.rb index cf869528..66312726 100644 --- a/lib/sqlite3/version_info.rb +++ b/lib/sqlite3/version_info.rb @@ -1,6 +1,5 @@ module SQLite3 - # a hash of descriptive metadata about the current version of the sqlite3 gem - VERSION_INFO = { + version_info = { ruby: RUBY_DESCRIPTION, gem: { version: SQLite3::VERSION @@ -14,4 +13,7 @@ module SQLite3 threadsafe: SQLite3.threadsafe? } } + # a hash of descriptive metadata about the current version of the sqlite3 gem, + # deeply frozen so that it can be read from any Ractor + VERSION_INFO = defined?(Ractor.make_shareable) ? Ractor.make_shareable(version_info) : version_info.freeze end diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb new file mode 100644 index 00000000..ec5a4cd4 --- /dev/null +++ b/test/test_integration_ractor.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +require "helper" +require "open3" +require "rbconfig" + +# Each Ractor test runs in a child process. Once a process has started a +# Ractor, some process-wide behavior changes for good (for example, +# ObjectSpace.each_object stops seeing objects), which would leak into +# unrelated tests. A child process also lets us fail on a hang instead of +# stalling the suite. +class IntegrationRactorTestCase < SQLite3::TestCase + TIMEOUT = 60 + + PRELUDE = <<~RUBY + require "sqlite3" + Warning[:experimental] = false + def ractor_value(r) = r.respond_to?(:value) ? r.value : r.take + RUBY + + def setup + super + skip("Requires a Ractor-safe build") unless SQLite3.ractor_safe? + end + + def test_ractor_safe + assert_predicate SQLite3, :ractor_safe? + end + + def test_open_and_query_in_ractors + assert_ractor_script(<<~RUBY, "4950,4950,4950,4950") + ractors = 4.times.map do + Ractor.new do + db = SQLite3::Database.new(":memory:") + begin + db.execute "CREATE TABLE t (a INTEGER)" + db.transaction do + stmt = db.prepare("INSERT INTO t VALUES (?)") + 100.times { |n| stmt.execute(n) } + stmt.close + end + db.get_first_value("SELECT sum(a) FROM t") + ensure + db.close + end + end + end + print ractors.map { ractor_value(_1) }.join(",") + RUBY + end + + def test_callbacks_in_ractor + assert_ractor_script(<<~RUBY, "20,b,true,true") + class Summer + def initialize = @sum = 0 + def step(v) = @sum += v + def finalize = @sum + end + + class Reverse + def compare(a, b) = b <=> a + end + + r = Ractor.new do + db = SQLite3::Database.new(":memory:") + begin + traced = false + db.trace { traced = true } + db.authorizer = ->(*) { true } + db.create_function("double", 1) { |fn, x| fn.result = x * 2 } + db.define_aggregator("summer", Summer.new) + db.collation("rev", Reverse.new) + + db.execute "CREATE TABLE t (a INTEGER, s TEXT)" + db.execute "INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'a'), (4, 'b')" + stmt = db.prepare("SELECT s FROM t ORDER BY s COLLATE rev LIMIT 1") + first = stmt.execute.to_a.flatten.first + stats = stmt.stat.key?(:vm_steps) + stmt.close + + [db.get_first_value("SELECT summer(double(a)) FROM t"), first, stats, traced].join(",") + ensure + db.close + end + end + print ractor_value(r) + RUBY + end + + def test_version_info_is_shareable + assert_ractor_script(<<~RUBY, SQLite3::VERSION) + print ractor_value(Ractor.new { SQLite3::VERSION_INFO[:gem][:version] }) + RUBY + end + + def test_database_cannot_be_sent_to_another_ractor + assert_ractor_script(<<~RUBY, "Ractor::Error") + db = SQLite3::Database.new(":memory:") + r = Ractor.new { Ractor.receive } + begin + r.send(db) + print "sent" + rescue Ractor::Error => e + print e.class + end + RUBY + end + + private + + def assert_ractor_script(script, expected) + load_path = $LOAD_PATH.select { |dir| File.directory?(dir) }.flat_map { |dir| ["-I", dir] } + cmd = [RbConfig.ruby, *load_path, "-e", PRELUDE + script] + + Open3.popen2e(*cmd) do |stdin, out, wait_thr| + stdin.close + reader = Thread.new { out.read } + + unless wait_thr.join(TIMEOUT) + Process.kill(:KILL, wait_thr.pid) + flunk "Ractor script did not finish within #{TIMEOUT}s:\n#{script}" + end + + output = reader.value + assert_predicate wait_thr.value, :success?, "Ractor script failed:\n#{output}" + assert_equal expected, output + end + end +end From 6bb743ce55fcf631dd80eab0e4500e97226cf320 Mon Sep 17 00:00:00 2001 From: Sam Ruby Date: Sat, 10 Oct 2026 08:53:34 -0400 Subject: [PATCH 2/4] Fix Ractor tests on Ruby 3.2 and under valgrind - Skip the send-a-Database test before Ruby 3.3, which made a shallow copy of T_DATA objects instead of raising (ruby/ruby@ce47ee00). - Compare only the child's stdout; stderr carries unrelated warnings, such as "Free at exit is experimental". - Unset RUBY_FREE_AT_EXIT for the child processes. With it set, Ruby 4.0.7 itself reads freed memory in ractor_free while tearing down any process that created a Ractor: RUBY_FREE_AT_EXIT=1 valgrind ruby -e 'Ractor.new { 1 }.value' Co-Authored-By: Claude Opus 5.5 --- test/test_integration_ractor.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb index ec5a4cd4..0d015e45 100644 --- a/test/test_integration_ractor.rb +++ b/test/test_integration_ractor.rb @@ -94,6 +94,9 @@ def test_version_info_is_shareable end def test_database_cannot_be_sent_to_another_ractor + # Before ruby/ruby@ce47ee00, sending a T_DATA object made a shallow copy instead of raising + skip("Requires Ruby 3.3 or later") if RUBY_VERSION < "3.3" + assert_ractor_script(<<~RUBY, "Ractor::Error") db = SQLite3::Database.new(":memory:") r = Ractor.new { Ractor.receive } @@ -112,17 +115,23 @@ def assert_ractor_script(script, expected) load_path = $LOAD_PATH.select { |dir| File.directory?(dir) }.flat_map { |dir| ["-I", dir] } cmd = [RbConfig.ruby, *load_path, "-e", PRELUDE + script] - Open3.popen2e(*cmd) do |stdin, out, wait_thr| + # RUBY_FREE_AT_EXIT (set by the valgrind task) makes Ruby itself read freed memory while + # tearing down a process that has created a Ractor, so it's unset for these child processes. + # Reproduce with: RUBY_FREE_AT_EXIT=1 valgrind ruby -e 'Ractor.new { 1 }.value' + env = {"RUBY_FREE_AT_EXIT" => nil} + + Open3.popen3(env, *cmd) do |stdin, out, err, wait_thr| stdin.close - reader = Thread.new { out.read } + out_reader = Thread.new { out.read } + err_reader = Thread.new { err.read } unless wait_thr.join(TIMEOUT) Process.kill(:KILL, wait_thr.pid) flunk "Ractor script did not finish within #{TIMEOUT}s:\n#{script}" end - output = reader.value - assert_predicate wait_thr.value, :success?, "Ractor script failed:\n#{output}" + output = out_reader.value + assert_predicate wait_thr.value, :success?, "Ractor script failed:\n#{output}#{err_reader.value}" assert_equal expected, output end end From f73f99f6c157549d4e51591cf2aa48a178042df2 Mon Sep 17 00:00:00 2001 From: Sam Ruby Date: Sat, 10 Oct 2026 09:00:05 -0400 Subject: [PATCH 3/4] Skip the Ractor tests under valgrind valgrind follows the child processes, and a Ruby process that has created a Ractor can't exit cleanly under it: with RUBY_FREE_AT_EXIT, Ruby 4.0.7 reads freed memory in ractor_free; without it, everything Ruby allocated is reported as leaked. Neither involves sqlite3. Co-Authored-By: Claude Opus 5.5 --- test/test_integration_ractor.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/test_integration_ractor.rb b/test/test_integration_ractor.rb index 0d015e45..3d2fa20a 100644 --- a/test/test_integration_ractor.rb +++ b/test/test_integration_ractor.rb @@ -21,6 +21,12 @@ def ractor_value(r) = r.respond_to?(:value) ? r.value : r.take def setup super skip("Requires a Ractor-safe build") unless SQLite3.ractor_safe? + + # valgrind follows the child processes, and a Ruby process that has created a Ractor can't + # exit cleanly under it: with RUBY_FREE_AT_EXIT, Ruby 4.0.7 reads freed memory in + # ractor_free (RUBY_FREE_AT_EXIT=1 valgrind ruby -e 'Ractor.new { 1 }.value'); without it, + # everything Ruby allocated is reported as leaked. + skip("Ractor processes can't exit cleanly under valgrind") if i_am_running_in_valgrind end def test_ractor_safe @@ -115,12 +121,7 @@ def assert_ractor_script(script, expected) load_path = $LOAD_PATH.select { |dir| File.directory?(dir) }.flat_map { |dir| ["-I", dir] } cmd = [RbConfig.ruby, *load_path, "-e", PRELUDE + script] - # RUBY_FREE_AT_EXIT (set by the valgrind task) makes Ruby itself read freed memory while - # tearing down a process that has created a Ractor, so it's unset for these child processes. - # Reproduce with: RUBY_FREE_AT_EXIT=1 valgrind ruby -e 'Ractor.new { 1 }.value' - env = {"RUBY_FREE_AT_EXIT" => nil} - - Open3.popen3(env, *cmd) do |stdin, out, err, wait_thr| + Open3.popen3(*cmd) do |stdin, out, err, wait_thr| stdin.close out_reader = Thread.new { out.read } err_reader = Thread.new { err.read } From ba52c995db537a68a15cb5fbfc364ac5093a570f Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Fri, 16 Oct 2026 10:27:05 -0700 Subject: [PATCH 4/4] Lets freeze constants We don't need to call make_shareable on everything :) --- ext/sqlite3/sqlite3.c | 4 ++-- lib/sqlite3/version_info.rb | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ext/sqlite3/sqlite3.c b/ext/sqlite3/sqlite3.c index 06e439e9..95f2ab6d 100644 --- a/ext/sqlite3/sqlite3.c +++ b/ext/sqlite3/sqlite3.c @@ -209,13 +209,13 @@ Init_sqlite3_native(void) rb_define_singleton_method(mSqlite3, "status", rb_sqlite3_status, -1); /* (String) The version of the sqlite3 library compiled with (e.g., "3.46.1") */ - rb_define_const(mSqlite3, "SQLITE_VERSION", rb_str_new2(SQLITE_VERSION)); + rb_define_const(mSqlite3, "SQLITE_VERSION", rb_obj_freeze(rb_str_new2(SQLITE_VERSION))); /* (Integer) The version of the sqlite3 library compiled with (e.g., 346001) */ rb_define_const(mSqlite3, "SQLITE_VERSION_NUMBER", INT2FIX(SQLITE_VERSION_NUMBER)); /* (String) The version of the sqlite3 library loaded at runtime (e.g., "3.46.1") */ - rb_define_const(mSqlite3, "SQLITE_LOADED_VERSION", rb_str_new2(sqlite3_libversion())); + rb_define_const(mSqlite3, "SQLITE_LOADED_VERSION", rb_obj_freeze(rb_str_new2(sqlite3_libversion()))); #ifdef USING_PACKAGED_LIBRARIES rb_define_const(mSqlite3, "SQLITE_PACKAGED_LIBRARIES", Qtrue); diff --git a/lib/sqlite3/version_info.rb b/lib/sqlite3/version_info.rb index 66312726..d2d5185e 100644 --- a/lib/sqlite3/version_info.rb +++ b/lib/sqlite3/version_info.rb @@ -1,9 +1,11 @@ module SQLite3 - version_info = { + # a hash of descriptive metadata about the current version of the sqlite3 gem, + # deeply frozen so that it can be read from any Ractor + VERSION_INFO = { ruby: RUBY_DESCRIPTION, gem: { - version: SQLite3::VERSION - }, + version: SQLite3::VERSION.freeze + }.freeze, sqlite: { compiled: SQLite3::SQLITE_VERSION, loaded: SQLite3::SQLITE_LOADED_VERSION, @@ -11,9 +13,6 @@ module SQLite3 precompiled: SQLite3::SQLITE_PRECOMPILED_LIBRARIES, sqlcipher: SQLite3.sqlcipher?, threadsafe: SQLite3.threadsafe? - } - } - # a hash of descriptive metadata about the current version of the sqlite3 gem, - # deeply frozen so that it can be read from any Ractor - VERSION_INFO = defined?(Ractor.make_shareable) ? Ractor.make_shareable(version_info) : version_info.freeze + }.freeze + }.freeze end