From 1b5ac51e666dafe7bb2b42eaf718526cc42c2ee6 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Fri, 20 Nov 2026 17:24:07 -0800 Subject: [PATCH 1/4] Allow 60 seconds for Arbitrum L1 progress --- docs/metrics.md | 10 ++++---- eth/payment_observer.go | 14 +++++++++++- eth/payment_observer_test.go | 44 ++++++++++++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 2c49a04..931c5cd 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -170,10 +170,12 @@ Other useful starting conditions: `/healthz` checks liveness. Paid orchestrator `/readyz` requires writable payment storage, chain access, an active recipient, valid pricing, and worker completion. Checks run at heartbeat cadence even with empty queues. Payment chain metadata -refreshes every five seconds. An observation is refused at thirty seconds old, -if the head timestamp is stale, or after thirty seconds without protocol L1 -progress. A successful RPC response with an unchanged head does not renew its -age. One provider cannot prove absolute L1 lag. Other dependency checks and +refreshes every five seconds. Observations and head timestamps expire after +30 seconds. Block progress must occur within 60 seconds when the first accepted +observation includes a separate L1 height, or 30 seconds when it uses the native +height. This deadline remains fixed for the observer's lifetime. Repeated heads +retain their original observation time. One provider cannot prove absolute L1 lag. +Other dependency checks and worker completion expire after `max(60 seconds, 3 × HeartbeatInterval)`; queue stalls and individual ticket errors remain separate signals. Off-chain orchestrator/discovery readiness reports listener health. diff --git a/eth/payment_observer.go b/eth/payment_observer.go index fea3b3b..f0884c3 100644 --- a/eth/payment_observer.go +++ b/eth/payment_observer.go @@ -19,6 +19,9 @@ import ( var ErrObservationUnavailable = errors.New("chain observation unavailable") +// Arbitrum's L1 height can stay flat across many fresh L2 heads. +const l1ProgressMaxAge = 60 * time.Second + // PaymentObserver publishes complete canonical observations. Gaps and reorgs // rebuild state at the new hash instead of replaying uncertain event history. type PaymentObserver struct { @@ -31,6 +34,7 @@ type PaymentObserver struct { header *miniHeader generation uint64 requireL1 bool + startedOnL1 bool contact, progress time.Time l1Progress time.Time l1HighWater *big.Int @@ -162,6 +166,10 @@ func (o *PaymentObserver) validate(s ChainSnapshot) error { func (o *PaymentObserver) viewRejection(s ChainSnapshot, l1Progress time.Time) string { now := o.now() + progressMaxAge := o.maxAge + if o.startedOnL1 { + progressMaxAge = l1ProgressMaxAge + } switch { case o.lifecycle != nil && o.lifecycle.Err() != nil: return "chain_observer_stopped" @@ -173,7 +181,7 @@ func (o *PaymentObserver) viewRejection(s ChainSnapshot, l1Progress time.Time) s return "chain_view_stale" case now.Sub(s.blockTime) >= o.maxAge: return "chain_head_stale" - case now.Sub(l1Progress) >= o.maxAge: + case now.Sub(l1Progress) >= progressMaxAge: return "chain_l1_progress_stale" default: return "" @@ -278,6 +286,10 @@ func (o *PaymentObserver) refresh(ctx context.Context) error { if o.viewRejection(s, l1Progress) != "" { return ErrObservationUnavailable } + if prior == nil { + // Tie the deadline to the clock that established the progress baseline. + o.startedOnL1 = head.L1BlockNumber != nil + } for key, entry := range o.payers { if _, affected := changed[key.payer]; reset || affected { entry.version++ diff --git a/eth/payment_observer_test.go b/eth/payment_observer_test.go index 8d66378..efb2044 100644 --- a/eth/payment_observer_test.go +++ b/eth/payment_observer_test.go @@ -169,16 +169,24 @@ func TestObserverRejectedHeadersInvalidateConflictingIdentity(t *testing.T) { } func TestObserverL1ProgressDeadline(t *testing.T) { - for _, clocks := range [][]uint64{{50, 50, 50}, {49, 50, 49}} { + elapsed := []time.Duration{10, 20, 30, 40, 50, 59, 60} + for _, tc := range []struct { + requireL1 bool + clocks [7]uint64 + }{ + {true, [7]uint64{50, 50, 50, 50, 50, 50, 50}}, + {false, [7]uint64{49, 50, 49, 50, 49, 50, 49}}, + } { f, o, now := newTestObserver(t) + o.requireL1 = tc.requireL1 require.NoError(t, o.refresh(t.Context())) progress, accepted := o.Status().L1ProgressAt, o.Status().ObservedAt - for step, clock := range clocks { - *now = now.Add(10 * time.Second) + for step, clock := range tc.clocks { + *now = progress.Add(elapsed[step] * time.Second) advanceObserver(f, *now) f.Update(func(s *test.ChainState) { s.L1Block = clock }) err := o.refresh(t.Context()) - if step < 2 { + if step < len(tc.clocks)-1 { require.NoError(t, err) accepted = o.Status().ObservedAt } else { @@ -196,6 +204,34 @@ func TestObserverL1ProgressDeadline(t *testing.T) { } } +func TestObserverNativeProgressDeadline(t *testing.T) { + for _, omitL1 := range []bool{true, false} { + f, o, now := newTestObserver(t) + o.requireL1 = false + f.Update(func(s *test.ChainState) { s.Number, s.OmitL1 = 100, true }) + require.NoError(t, o.refresh(t.Context())) + + *now = now.Add(29 * time.Second) + f.Update(func(s *test.ChainState) { + s.Hash, s.Timestamp = common.HexToHash("0xbeef"), uint64(now.Unix()) + s.L1Block, s.OmitL1 = 50, omitL1 + }) + require.NoError(t, o.refresh(t.Context())) + require.True(t, o.Status().Healthy) + + *now = now.Add(time.Second) + require.ErrorIs(t, o.refresh(t.Context()), ErrObservationUnavailable) + require.Equal(t, "chain_l1_progress_stale", o.Status().RecipientRejection) + + if omitL1 { + *now = now.Add(5 * time.Second) + advanceObserver(f, *now) + f.Update(func(s *test.ChainState) { s.L1Block, s.OmitL1 = 50, false }) + require.ErrorIs(t, o.refresh(t.Context()), ErrObservationUnavailable) + } + } +} + func TestObserverExpiredCandidateIsNotPublished(t *testing.T) { for _, phase := range []string{"RoundsManager.currentRound", "eth_getLogs"} { t.Run(phase, func(t *testing.T) { From 5629212f1dcefacfaa7908dbe5169e39bfd8b5ce Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Fri, 20 Nov 2026 17:32:02 -0800 Subject: [PATCH 2/4] Clarify payment chain freshness limits --- docs/metrics.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 931c5cd..decaa00 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -171,13 +171,11 @@ Other useful starting conditions: storage, chain access, an active recipient, valid pricing, and worker completion. Checks run at heartbeat cadence even with empty queues. Payment chain metadata refreshes every five seconds. Observations and head timestamps expire after -30 seconds. Block progress must occur within 60 seconds when the first accepted -observation includes a separate L1 height, or 30 seconds when it uses the native -height. This deadline remains fixed for the observer's lifetime. Repeated heads -retain their original observation time. One provider cannot prove absolute L1 lag. -Other dependency checks and -worker completion expire after `max(60 seconds, 3 × HeartbeatInterval)`; -queue stalls and individual ticket errors remain separate signals. +30 seconds. Block height must advance within 30 seconds, or within 60 seconds +if the observer starts with a separate L1 height. One provider cannot prove +absolute L1 lag. Other dependency checks and worker completion expire after +`max(60 seconds, 3 × HeartbeatInterval)`. Queue stalls and individual ticket +errors remain separate signals. Off-chain orchestrator/discovery readiness reports listener health. Signer readiness requires chain access, positive deposit/total reserve, no withdrawal through the next round, fresh pricing, and healthy enabled outbox From 3ba4b688961942f4e0eb4a3c607104bbeec955e3 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Fri, 20 Nov 2026 18:29:30 -0800 Subject: [PATCH 3/4] Simplify Arbitrum readiness wording --- docs/metrics.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index decaa00..945f9af 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -169,11 +169,10 @@ Other useful starting conditions: `/healthz` checks liveness. Paid orchestrator `/readyz` requires writable payment storage, chain access, an active recipient, valid pricing, and worker completion. -Checks run at heartbeat cadence even with empty queues. Payment chain metadata -refreshes every five seconds. Observations and head timestamps expire after -30 seconds. Block height must advance within 30 seconds, or within 60 seconds -if the observer starts with a separate L1 height. One provider cannot prove -absolute L1 lag. Other dependency checks and worker completion expire after +Checks run at heartbeat cadence even with empty queues. Chain data refreshes +every five seconds. On Arbitrum, L2 observations expire after 30 seconds and L1 +height must advance within 60 seconds. One provider cannot prove absolute L1 +lag. Other dependency checks and worker completion expire after `max(60 seconds, 3 × HeartbeatInterval)`. Queue stalls and individual ticket errors remain separate signals. Off-chain orchestrator/discovery readiness reports listener health. From 6fb79a3f1f7508b98f5a7229f97d52899ab38b34 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Fri, 20 Nov 2026 18:31:01 -0800 Subject: [PATCH 4/4] Bump version to 1.0.10 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 66c4c22..7ee7020 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.9 +1.0.10