From 98e4a72d384b37ff863cbec0bd710c48c4b00e53 Mon Sep 17 00:00:00 2001 From: Drew Robinson Date: Fri, 18 Sep 2026 09:29:43 +1000 Subject: [PATCH 1/2] fix: perform initial sync when opening an embedded replica connect/1 built the replica and began serving reads without ever pulling from the primary. LibSQL pushes local writes upstream on its own, but it never pulls remote changes unless sync() is called, so a freshly opened replica read from an empty or stale local file until the caller invoked EctoLibSql.Native.sync/1 by hand. The :sync option was threaded from connect/1 all the way down to the NIF and then discarded, so nothing acted on it. It is now read in connect and an initial sync runs before the first connection is handed out. Also corrects the README, which claimed remote changes are pulled in the background. They are not - sync/1 is still required to observe changes made after connecting. Fixes #118 Claude-Session: https://claude.ai/code/session_01QeALzUVYXSgNWcYMEJnqeY --- CHANGELOG.md | 10 +++++- README.md | 8 +++-- native/ecto_libsql/src/connection.rs | 14 ++++++++ test/turso_remote_test.exs | 50 ++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c137930..bd21c6cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.9.1] - 2026-05-07 +## [Unreleased] + +### Fixed + +- **Embedded Replica Initial Sync** - `connect/1` now performs an initial sync before serving any reads when an embedded replica is opened with `sync: true`. LibSQL pushes local writes to the primary on its own, but it never pulls remote changes unless `sync/1` is called, so a freshly opened replica read from an empty or stale local file - contradicting the README's claim that initial sync happens when you first connect. (Reported in [#118](https://github.com/ocean/ecto_libsql/issues/118)) + +### Changed + +- **Replica Sync Documentation** - Corrected the README's description of automatic sync. Remote changes made *after* you connect are not pulled in the background; `sync/1` is still required to observe them. ### Fixed diff --git a/README.md b/README.md index 7b1e7a0d..bf26b335 100644 --- a/README.md +++ b/README.md @@ -363,9 +363,11 @@ EctoLibSql.handle_execute("SELECT * FROM users", [], [], state) ``` **How automatic sync works:** -- Initial sync happens when you first connect -- Changes are synced automatically in the background -- You don't need to call `sync/1` in most applications +- An initial sync runs when you first connect, so the replica starts from the current + state of the primary rather than an empty or stale local file +- Your own writes are pushed to the primary automatically as you make them +- Remote changes made *after* you connect are **not** pulled in the background - call + `sync/1` when you need to observe them #### Manual Sync Control diff --git a/native/ecto_libsql/src/connection.rs b/native/ecto_libsql/src/connection.rs index 4c0110b0..b76e36a7 100644 --- a/native/ecto_libsql/src/connection.rs +++ b/native/ecto_libsql/src/connection.rs @@ -62,6 +62,10 @@ pub fn connect(opts: Term, mode: Term) -> NifResult { let remote_encryption_key = map .get("remote_encryption_key") .and_then(|t| t.decode::().ok()); + let sync_enabled = map + .get("sync") + .and_then(|t| t.decode::().ok()) + .unwrap_or(false); // Wrap the entire connection process with a timeout using the global runtime. TOKIO_RUNTIME.block_on(async { @@ -136,6 +140,16 @@ pub fn connect(opts: Term, mode: Term) -> NifResult { } .map_err(|e| rustler::Error::Term(Box::new(format!("Failed to build DB: {e}"))))?; + // Pull the current state of the primary before serving any reads. LibSQL pushes + // writes to the primary on its own, but it never pulls remote changes unless + // sync() is called, so without this a freshly opened replica reads from an empty + // or stale local file. + if mode_enum == Mode::RemoteReplica && sync_enabled { + db.sync().await.map_err(|e| { + rustler::Error::Term(Box::new(format!("Failed initial sync: {e}"))) + })?; + } + let conn = db .connect() .map_err(|e| rustler::Error::Term(Box::new(format!("Failed to connect: {e}"))))?; diff --git a/test/turso_remote_test.exs b/test/turso_remote_test.exs index ce1a3da3..6b9c213a 100644 --- a/test/turso_remote_test.exs +++ b/test/turso_remote_test.exs @@ -719,6 +719,56 @@ defmodule TursoRemoteTest do end describe "embedded replica with sync" do + test "a fresh replica sees existing remote rows without a manual sync", %{table_name: table} do + # Regression test for #118. LibSQL pushes local writes to the primary on its own, + # but it never pulls remote changes unless sync() is called, so before the initial + # sync was added to connect/1 a freshly opened replica read from an empty local + # file and returned nothing here. + {:ok, remote_state} = EctoLibSql.connect(uri: @turso_uri, auth_token: @turso_token) + + {:ok, _, _, remote_state} = + EctoLibSql.handle_execute( + "CREATE TABLE IF NOT EXISTS #{table} (id INTEGER PRIMARY KEY, value TEXT)", + [], + [], + remote_state + ) + + {:ok, _, _, remote_state} = + EctoLibSql.handle_execute( + "INSERT INTO #{table} (id, value) VALUES (?, ?)", + [1, "written_before_replica_existed"], + [], + remote_state + ) + + EctoLibSql.disconnect([], remote_state) + + local_db = "z_ecto_libsql_test-initial_sync_#{:erlang.unique_integer([:positive])}.db" + on_exit(fn -> cleanup_local_db(local_db) end) + + {:ok, replica_state} = + EctoLibSql.connect( + database: local_db, + uri: @turso_uri, + auth_token: @turso_token, + sync: true + ) + + # Read immediately, with no intervening EctoLibSql.Native.sync/1 call. + {:ok, _, result, replica_state} = + EctoLibSql.handle_execute( + "SELECT value FROM #{table} WHERE id = ?", + [1], + [], + replica_state + ) + + assert result.rows == [["written_before_replica_existed"]] + + EctoLibSql.disconnect([], replica_state) + end + test "automatic sync from local to remote", %{table_name: table} do # Create unique local database file for this test local_db = "z_ecto_libsql_test-replica_#{:erlang.unique_integer([:positive])}.db" From e9d775f454290a806540a6318c014828f253e985 Mon Sep 17 00:00:00 2001 From: Drew Robinson Date: Fri, 18 Sep 2026 11:41:26 +1000 Subject: [PATCH 2/2] docs: note that mix test does not exercise local Rust changes The NIF is loaded through RustlerPrecompiled, so mix test uses the downloaded artefact for the current version and silently ignores edits under native/ecto_libsql/src. A Rust change can look entirely green locally while never having been loaded - which is how the fix in this branch first appeared not to work. Records the three things needed to test Rust from the Elixir suite: ECTO_LIBSQL_BUILD=1, a forced recompile per MIX_ENV, and a larger dirty-IO scheduler stack. The last one matters because native.ex selects a debug build for dev and test, and debug-built libSQL overflows the default stack and aborts the VM with SIGBUS, usually on the first remote connection so it reads as an unrelated failure. Claude-Session: https://claude.ai/code/session_01QeALzUVYXSgNWcYMEJnqeY --- CLAUDE.md | 21 +++++++++++++++++++++ TESTING.md | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index af9172bd..b0a0a9c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -317,6 +317,27 @@ cd native/ecto_libsql && cargo test -- --nocapture # Rust output with stdout for i in {1..10}; do mix test test/file.exs:42; done # Flush out race conditions ``` +### ⚠️ `mix test` does NOT exercise your Rust changes + +The NIF is loaded via `RustlerPrecompiled`, so `mix test` uses the **downloaded** artefact +for the current version and silently ignores edits under `native/ecto_libsql/src`. Rust +changes can appear fully green while never having been loaded. To test them for real: + +```bash +export ECTO_LIBSQL_BUILD=1 # force a source build +MIX_ENV=test mix compile --force # --force required; plain `mix compile` no-ops +ERL_FLAGS="+sssdio 8192" mix test # debug builds need a larger dirty-IO stack +``` + +- `--force` is required: without a changed Elixir file Mix skips the crate build entirely. +- `_build/dev` and `_build/test` hold separate NIFs - rebuild for the env you are using. +- `ERL_FLAGS="+sssdio 8192"` avoids a `SIGBUS` (exit 138) VM abort. `native.ex` selects + `mode: :debug` for dev/test and debug-built libSQL overflows the default dirty-IO + scheduler stack, usually on the first remote connection, so it looks unrelated. + +Check for `Compiling crate ecto_libsql` in the output; `Copying NIF from cache` means you +are still on the precompiled artefact. See [TESTING.md](TESTING.md) for detail. + ### Test Variable Naming Conventions Use consistent variable names by scope: diff --git a/TESTING.md b/TESTING.md index 4496e0f5..2a045c54 100644 --- a/TESTING.md +++ b/TESTING.md @@ -362,6 +362,47 @@ test tests::registry_tests::test_uuid_generation ... ok test result: ok. 19 passed; 0 failed; 0 ignored ``` +### ⚠️ Exercising Rust changes from the Elixir suite + +`cargo test` covers the Rust in isolation, but `mix test` does **not** pick up your Rust +changes by default. `EctoLibSql.Native` is built on `RustlerPrecompiled`, so the NIF is +downloaded from the release artefacts for the current version, and an edit to +`native/ecto_libsql/src` is silently ignored. A change can look completely green locally +while never having been loaded. + +To run the Elixir suite against locally built Rust: + +```bash +export ECTO_LIBSQL_BUILD=1 # force a source build instead of the precompiled NIF +MIX_ENV=test mix compile --force # --force is required; plain `mix compile` no-ops +ERL_FLAGS="+sssdio 8192" mix test +``` + +Three things to watch: + +- **`--force` is not optional.** Setting `ECTO_LIBSQL_BUILD` alone is not enough, because + Mix sees no changed Elixir files and skips the compile that triggers the crate build. +- **Each `MIX_ENV` has its own NIF.** `_build/dev` and `_build/test` hold separate copies, + so building for one leaves the other stale. Rebuild for the env you are about to use. +- **`ERL_FLAGS="+sssdio 8192"` is needed for local builds.** `lib/ecto_libsql/native.ex` + selects `mode: :debug` for `:dev` and `:test`, and debug-built libSQL overflows the + default dirty-IO scheduler stack, aborting the VM with `SIGBUS` (exit 138) - typically + on the first remote connection, which makes it look like an unrelated failure. These + NIFs run with `schedule = "DirtyIo"`, so `+sssdio` is the stack size that matters. The + precompiled artefacts are release builds and do not need this. + +Confirm you are actually running your own build - this line means the crate compiled: + +``` +Compiling crate ecto_libsql in debug mode (native/ecto_libsql) +``` + +whereas this line means you are still on the precompiled NIF: + +``` +[debug] Copying NIF from cache and extracting to .../libecto_libsql-vX.Y.Z-....so +``` + ### Elixir Tests ```bash