Ensure Non-Validators and Validators Bootstrap Before Indexing and Verifying Blocks by samliok · Pull Request #573 · ava-labs/Simplex

samliok · 2026-09-01T22:56:52Z

Addresses #530 but for non-validator and validators.

Nodes will first need to complete bootstrapping before being able to index/verify blocks.

yacovm · 2026-09-09T19:51:37Z

Looks like this test demonstrates we have a liveness problem in case we miss the first broadcast we send via broadcastLatestEpoch:

// TestNonValidatorBootstrapsWhenStartupBroadcastIsLost asserts a non-validator still bootstraps when
// nobody receives the replication request it broadcasts on start. The non-validator joins an empty
// network, so that broadcast reaches no one, and the validator only comes online afterwards. Until it
// bootstraps, the non-validator drops every block and finalization the validator sends it, so the only
// way for it to recover is to ask again.
func TestNonValidatorBootstrapsWhenStartupBroadcastIsLost(t *testing.T) {
	validator := newNodeMapping(1)
	pChain := newTestPChain([]metadata.NodeBLSMapping{validator})
	network := newNetwork(t, pChain)

	// The non-validator starts alone, so the replication request it broadcasts on start is lost.
	nonValidator := newNodeMapping(2)
	node := network.addNode(nonValidator.NodeID[:])
	isValidator, bootstrapped := node.role()
	require.False(t, isValidator)
	require.False(t, bootstrapped)

	// The validator then comes online and commits the first block on its own,
	// broadcasting its finalization to the non-validator, which drops it.
	network.addNode(validator.NodeID[:]).sync()

	// The validator is there to answer, so the non-validator must eventually ask again and bootstrap.
	require.Eventually(t, func() bool {
		_, bootstrapped := node.role()
		return bootstrapped
	}, 20*time.Second, 100*time.Millisecond, "the non-validator never bootstrapped after its startup broadcast was lost")

	node.sync()
}

yacovm · 2026-09-09T20:00:35Z

In the issue #530 I wrote:

What we should do instead is when we bootstrap the node, we replicate only the sealing blocks backwards, until we reach the latest committed sealing block, and then we keep the chain of sealing blocks and use them and never replicate them by verifying quorum certificates.

The below test demonstrates that that's not what we do:

// TestNonValidatorRejectsSealingBlockOffTheHashChain asserts a bootstrapping non-validator only accepts
// sealing blocks that lie on the backward hash chain from the sealing block it bootstrapped on, however
// well they are signed. See https://github.com/ava-labs/Simplex/issues/530.
//
// The node has committed epoch 1's defining block. The current validator set tells it the latest sealing
// block is at seq 3, whose PrevSealingBlockHash names an honest sealing block at seq 2. A peer then serves
// a forged sealing block at seq 2 that carries a valid quorum certificate of epoch 1's validators but is
// not the block seq 3 points to. Epoch 1's keys may have leaked long ago, so the forged block must be
// rejected. Today the node validates it by its quorum certificate alone and indexes it.
func TestNonValidatorRejectsSealingBlockOffTheHashChain(t *testing.T) {
	validator := newNodeMapping(1)
	validators := metadata.NodeBLSMappings{validator}
	signers := []common.NodeID{validator.NodeID[:]}
	sigAggregator := &testutil.TestSignatureAggregator{N: len(validators)}

	pChain := newTestPChain(validators)
	storage, epochBlock := newChainStorage(t, validators)

	// A node outside the validator set whose ledger ends at epoch 1's defining block.
	nonValidator := newNodeMapping(2)
	node := newNetwork(t, pChain).addNodeWithConfig(nonValidator.NodeID[:], nodeConfig{storage: storage})
	isValidator, bootstrapped := node.role()
	require.False(t, isValidator)
	require.False(t, bootstrapped)

	// sealingBlock builds a sealing block at seq in the given epoch, keeping the same validator set.
	sealingBlock := func(seq, epoch uint64, prev, prevSealingBlockHash [32]byte, payload string) *ParsedBlock {
		timestamp := epochBlockTime.Add(time.Duration(seq) * time.Millisecond)
		return &ParsedBlock{StateMachineBlock: metadata.StateMachineBlock{
			InnerBlock: &testInnerBlock{Height_: seq, TS: timestamp, Payload: []byte(payload)},
			Metadata: metadata.StateMachineMetadata{
				Timestamp:               uint64(timestamp.UnixMilli()),
				SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: epoch, Round: seq, Seq: seq, Prev: common.Digest(prev)},
				SimplexEpochInfo: metadata.SimplexEpochInfo{
					EpochNumber:          epoch,
					PrevSealingBlockHash: prevSealingBlockHash,
					BlockValidationDescriptor: &metadata.BlockValidationDescriptor{
						AggregatedMembership: metadata.AggregatedMembership{Members: validators},
					},
				},
			},
		}}
	}

	// The honest chain: seq 1 (in storage) <- honest seq 2 <- seq 3. The forged seq 2 is signed by the
	// same epoch 1 validators but is not what seq 3 points to.
	honestSealing2 := sealingBlock(2, 1, epochBlock.Digest(), epochBlock.Digest(), "honest sealing block 2")
	forgedSealing2 := sealingBlock(2, 1, epochBlock.Digest(), epochBlock.Digest(), "forged sealing block 2")
	require.NotEqual(t, honestSealing2.Digest(), forgedSealing2.Digest())
	sealing3 := sealingBlock(3, 2, honestSealing2.Digest(), honestSealing2.Digest(), "sealing block 3")

	honestFinalization2, _ := testutil.NewFinalizationRecord(t, sigAggregator, honestSealing2, signers)
	forgedFinalization2, _ := testutil.NewFinalizationRecord(t, sigAggregator, forgedSealing2, signers)
	finalization3, _ := testutil.NewFinalizationRecord(t, sigAggregator, sealing3, signers)

	// The current validator set reports seq 3 as the latest sealing block, which bootstraps the node.
	require.NoError(t, node.inst.HandleMessage(&common.Message{
		ReplicationResponse: &common.ReplicationResponse{
			LatestSeq: &common.QuorumRound{Block: sealing3, Finalization: &finalization3},
		},
	}, validator.NodeID[:]))
	_, bootstrapped = node.role()
	require.True(t, bootstrapped)

	// A peer serves the forged seq 2. Its quorum certificate checks out against epoch 1, but its digest
	// is not the PrevSealingBlockHash of seq 3, so it is off the hash chain and must not be indexed.
	require.NoError(t, node.inst.HandleMessage(&common.Message{
		ReplicationResponse: &common.ReplicationResponse{
			Data: []common.QuorumRound{{Block: forgedSealing2, Finalization: &forgedFinalization2}},
		},
	}, validator.NodeID[:]))
	require.Never(t, func() bool {
		_, _, err := node.storage.Retrieve(2)
		return err == nil
	}, 2*time.Second, 50*time.Millisecond, "indexed a sealing block that is not on the hash chain from the sealing block we bootstrapped on")

	// The honest seq 2 is what seq 3 points to, so it is accepted.
	require.NoError(t, node.inst.HandleMessage(&common.Message{
		ReplicationResponse: &common.ReplicationResponse{
			Data: []common.QuorumRound{{Block: honestSealing2, Finalization: &honestFinalization2}},
		},
	}, validator.NodeID[:]))
	committed := node.storage.WaitForBlockCommit(2)
	require.Equal(t, honestSealing2.Bytes(), committed.Bytes())
}

yacovm · 2026-09-09T20:02:49Z

+		return nil
+	}
+
+	n.Logger.Info("Bootstrapped, received a threshold of sealing block info for an epoch", zap.Stringer("Info", qr.Block.SealingBlockInfo()))


We should only consider ourselves as bootstrapped when we have replicated and committed all blocks from the last block in the ledger to the last known tip.

i think we can consider ourselves bootstrapped once we have validated the hash chain of sealing blocks, then we can start as normal syncing all the blocks in between

as to your test, i made a few non-validator tests to ensure we do the backwards hash validation first

TestNonValidator_BootstrapIgnoresSealingBlockOffChain && TestNonValidator_BootstrapWalksHashChain

yacovm

These are the comments I have so far, I'm not nearly done with the review.

yacovm · 2026-09-11T18:58:02Z

 	// and it is in the validator set
 	TransitionToValidator func(epoch uint64, validators common.Nodes)
+
+	// Bootstrapped is set once every epoch from our tip up to the one a threshold of the latest


Why would we call that bootstrapped? Bootstrapped should just mean that we finished bootstrapping, exactly like we do in snowman. Which is that we have replicated all blocks we know are missing from the latest discovered tip down to the tip before bootstrapping.

updated from bootstrapped terminology to EpochsReplicated 57c282e

yacovm · 2026-09-11T18:59:07Z

+		return nil
+	}
+
+	// No sealing block is missing, so every epoch from our tip to the highest is validated.


but we should still replicate all blocks in the last epoch that we know about before we declare that we have finished bootstrapping.

I think this comment is still relevant

updated from bootstrapped terminology to EpochsReplicated 57c282e

yacovm · 2026-09-11T19:28:24Z

 	}

-	comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, mappings.Nodes())
+	comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, latestValidatorSet.Nodes())


unrelated to this PR, but... what updates the comm's validator set once we move through epochs after we bootstrap?

yea we need to change the non-validator comm to not hardcode its Validators() method. everytime it calls Validators the pchain should be queried or something

yacovm · 2026-09-28T17:23:44Z

+	for _, seq := range seqs {
+		n.Logger.Debug("Re-requesting a sealing block", zap.Uint64("Seq", seq))
+		n.Comm.Broadcast(&common.Message{
+			ReplicationRequest: &common.ReplicationRequest{Seqs: []uint64{seq}},


shouldn't we add here the LatestFinalizedSeq or call broadcastLatestEpoch?

Seems like the initial broadcastLatestEpoch we do in Start() only works once now, no?

ah yea good point

yacovm · 2026-09-28T20:17:57Z

+
+	n.sealingBlockTimeouts.RemoveTask(nextEpoch)
+
+	// The first simplex block opens its own epoch, so there is no earlier sealing block.


how is this comment relevant to the code below it?

Is it about prevSealingSeq == nextEpoch ?

yacovm · 2026-09-28T20:21:02Z

+	}
+}
+
+// validateSealingBlock validates the epoch a sealing block opens and stores its quorum round.


I am not sure validateX is the right way to call this method.

Judging by the name I would expect it should check the sealing block and return if it's valid or not.

In practice the method stores it or even sends request to fetch the previous one.

Can we give it a more descriptive name?

Perhaps we can call this processSealingBlock and call maybeValidateNextEpoch - maybeReplicateNextEpoch ?

And can we add a comment saying that we expect the given block to have already been authenticated here and in maybeValidateNextEpoch?

yacovm · 2026-09-28T20:30:07Z


-func (n *NonValidator) maybeValidateNextEpoch(block common.Block) {
-	nextEpoch := block.BlockHeader().Seq
+// maybeValidateNextEpoch validates the epoch block opens when block is a sealing block. While


maybeValidateNextEpoch validates the epoch block opens when block is a sealing block.

I can't grok this sentence. The epoch block opens what?

yacovm · 2026-09-28T20:53:23Z

 		if block.SealingBlockInfo() != nil {
-			highestEpoch, highestValidatorSet := n.epochs.highestEpoch()
-
-			// We should only transition to become a validator, if the sealing block is creating the highest


did you fold the below lines into maybeTransitionToValidator ?

yacovm · 2026-09-28T21:10:49Z

+		if !n.isIndexed(bh.Seq) {
+			n.validateSealingBlock(qr, from)
+		}
+	case n.highestEpochCollector.collectedSealingBlockInfo(sealingInfo, bh, from):


maybe we can rename collectedSealingBlockInfo to something that reflects better its implementation or what it actually stands for?

Perhaps maybeObserveThresholdResponses() ?

yacovm · 2026-09-28T21:23:28Z

 }

+func (i *Instance) maybeReplicateEpochs() error {
+	i.Config.Logger.Debug("Checking if epoch replication is required")


Shouldn't it be something like: "Checking if latest persisted validator set is up to date" ?

yacovm · 2026-09-28T21:24:54Z

+	// We have indexed the latest validator set, therefore we can skip epoch replication and start as a validator.
+	// Note: this may not be the latest epoch, but a future PR will eventually notice we are behind and transition properly.
+	if latestIndexedEpochValidators.Equal(latestValidatorSet.Nodes()) && latestValidatorSet.Nodes().Contains(i.Config.ID) {
+		i.Config.Logger.Debug("Node skipping epoch replication because its latest epoch is up to date with the Platform Chain")


We should prefer writing log messages that are understandable to node operators and not to us.

So instead of "epoch replication" let's say "validator set replication" here and below.

I think it conveys more what we're trying to accomplish.

yacovm · 2026-09-28T21:40:26Z

+	network.setOnline(v3.NodeID[:])
+	node.sync()
+
+	// The only way for the epoch transition to finish is if ourNode becomes a validator


shouldn't we check the node is also a validator now?

yacovm · 2026-09-28T21:53:36Z

+}
+
+// TestNonValidator_EpochReplicationWalksHashChain asserts a non-validator several epochs behind requests
+// sealing blocks one hop back at a time once a threshold reports the highest one, and finishes


why "once" and not "starting from" ?

yacovm · 2026-09-28T21:56:09Z

+}
+
+// TestNonValidator_EpochReplicationIgnoresSealingBlockOffChain asserts that while following the hash chain
+// a sealing block further down it is dropped until the epoch pointing back to it has been validated.


a sealing block further down it is dropped until the epoch pointing back to it has been validated

I am not sure I understand what this means. Why is a sealing block dropped? How can something be dropped until something?

yacovm · 2026-09-28T22:18:02Z

+	require.NoError(t, err)
+	defer nv.Stop()
+
+	inOurEpoch := tc.appendBlock()


The code below is confusing regarding which epoch corresponds to which block.

Can we add some kind of require.Equal(expectedEpoch, block.Epoch) after to make this more clear?

yacovm · 2026-09-28T22:29:39Z

@@ -1047,6 +1058,291 @@ func TestNonValidatorRejectsQuorumRoundFromNonValidator(t *testing.T) {
 	}


Aren't we missing a non_validator_test test case that shows that when a new non-validator is added, it replicates the sealing blocks first and afterwards the blocks between them?

samliok self-assigned this Sep 1, 2026

samliok force-pushed the bootstrap-first branch 3 times, most recently from 6be0d9c to 23d9c67 Compare September 1, 2026 23:55

samliok linked an issue Sep 2, 2026 that may be closed by this pull request

Validate sealing blocks only via backward hash chain validation on bootstrap instead of also via QC #530

Open

samliok force-pushed the bootstrap-first branch 2 times, most recently from 9c76fa7 to 9501164 Compare September 2, 2026 16:18

samliok force-pushed the bootstrap-first branch from 9501164 to a1bb893 Compare September 2, 2026 18:11

samliok force-pushed the bootstrap-first branch from 9d1886a to b4bfaba Compare September 3, 2026 23:02

Base automatically changed from instance-test-refactor to main September 4, 2026 17:58

yacovm force-pushed the bootstrap-first branch from b4bfaba to 4b6b2f5 Compare September 4, 2026 17:58

samliok force-pushed the bootstrap-first branch 6 times, most recently from a1a3fb9 to b507b64 Compare September 9, 2026 18:30

yacovm reviewed Sep 9, 2026

View reviewed changes

Comment thread nonvalidator/epochs.go Outdated

yacovm reviewed Sep 9, 2026

View reviewed changes

samliok force-pushed the bootstrap-first branch from b04bddb to c2d3997 Compare September 10, 2026 01:20

samliok marked this pull request as draft September 10, 2026 14:25

samliok marked this pull request as ready for review September 10, 2026 18:45

yacovm reviewed Sep 11, 2026

View reviewed changes

samliok added 13 commits September 28, 2026 15:55

bootstrap before verifying and indexing blocks

67968e5

bootstrapping reworked

5d88997

lint

57e9677

reduce diff

c37a757

reduce more diff

70ed26b

add offline test

af85780

lint

2fc3863

comments

c3cd079

bootstrap

b8bc87e

one source of truth for storage

8dae965

add comments and is empty

175f0cc

bootstrap rename to epochs replicated

e43b1c3

update comment and reording

3e51cb0

samliok force-pushed the bootstrap-first branch from 1751db3 to 3e51cb0 Compare September 28, 2026 14:59

yacovm reviewed Sep 28, 2026

View reviewed changes

samliok added 3 commits September 28, 2026 18:42

add latest seq request

4c3ce88

initial replication task of 0

dda7b7d

comment nits

b6bc91e

yacovm reviewed Sep 28, 2026

View reviewed changes


		n.sealingBlockTimeouts.RemoveTask(nextEpoch)

		// The first simplex block opens its own epoch, so there is no earlier sealing block.

		@@ -1047,6 +1058,291 @@ func TestNonValidatorRejectsQuorumRoundFromNonValidator(t *testing.T) {
		}

Conversation

samliok commented Sep 1, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

Uh oh!

yacovm commented Sep 9, 2026

Uh oh!

yacovm commented Sep 9, 2026

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

yacovm left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

samliok Sep 11, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

yacovm Sep 28, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

samliok Sep 28, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

yacovm Sep 28, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Reviewers

samliok commented Sep 1, 2026 •

edited

Loading

samliok Sep 11, 2026 •

edited

Loading

yacovm Sep 28, 2026 •

edited

Loading

samliok Sep 28, 2026 •

edited

Loading

yacovm Sep 28, 2026 •

edited

Loading