Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.0.9
1.0.10
13 changes: 6 additions & 7 deletions docs/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -169,13 +169,12 @@ Other useful starting conditions:

`/healthz` checks liveness. Paid orchestrator `/readyz` requires writable payment
storage, chain access, an active recipient, valid pricing, and worker completion.
Checks run at heartbeat cadence even with empty queues. Payment chain metadata
refreshes every five seconds. An observation is refused at thirty seconds old,
if the head timestamp is stale, or after thirty seconds without protocol L1
progress. A successful RPC response with an unchanged head does not renew its
age. One provider cannot prove absolute L1 lag. Other dependency checks and
worker completion expire after `max(60 seconds, 3 × HeartbeatInterval)`;
queue stalls and individual ticket errors remain separate signals.
Checks run at heartbeat cadence even with empty queues. Chain data refreshes
every five seconds. On Arbitrum, L2 observations expire after 30 seconds and L1
height must advance within 60 seconds. One provider cannot prove absolute L1
lag. Other dependency checks and worker completion expire after
`max(60 seconds, 3 × HeartbeatInterval)`. Queue stalls and individual ticket
errors remain separate signals.
Off-chain orchestrator/discovery readiness reports listener health.
Signer readiness requires chain access, positive deposit/total reserve, no
withdrawal through the next round, fresh pricing, and healthy enabled outbox
Expand Down
14 changes: 13 additions & 1 deletion eth/payment_observer.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ import (

var ErrObservationUnavailable = errors.New("chain observation unavailable")

// Arbitrum's L1 height can stay flat across many fresh L2 heads.
const l1ProgressMaxAge = 60 * time.Second

// PaymentObserver publishes complete canonical observations. Gaps and reorgs
// rebuild state at the new hash instead of replaying uncertain event history.
type PaymentObserver struct {
Expand All @@ -31,6 +34,7 @@ type PaymentObserver struct {
header *miniHeader
generation uint64
requireL1 bool
startedOnL1 bool
contact, progress time.Time
l1Progress time.Time
l1HighWater *big.Int
Expand Down Expand Up @@ -162,6 +166,10 @@ func (o *PaymentObserver) validate(s ChainSnapshot) error {

func (o *PaymentObserver) viewRejection(s ChainSnapshot, l1Progress time.Time) string {
now := o.now()
progressMaxAge := o.maxAge
if o.startedOnL1 {
progressMaxAge = l1ProgressMaxAge
}
switch {
case o.lifecycle != nil && o.lifecycle.Err() != nil:
return "chain_observer_stopped"
Expand All @@ -173,7 +181,7 @@ func (o *PaymentObserver) viewRejection(s ChainSnapshot, l1Progress time.Time) s
return "chain_view_stale"
case now.Sub(s.blockTime) >= o.maxAge:
return "chain_head_stale"
case now.Sub(l1Progress) >= o.maxAge:
case now.Sub(l1Progress) >= progressMaxAge:
return "chain_l1_progress_stale"
default:
return ""
Expand Down Expand Up @@ -278,6 +286,10 @@ func (o *PaymentObserver) refresh(ctx context.Context) error {
if o.viewRejection(s, l1Progress) != "" {
return ErrObservationUnavailable
}
if prior == nil {
// Tie the deadline to the clock that established the progress baseline.
o.startedOnL1 = head.L1BlockNumber != nil
}
for key, entry := range o.payers {
if _, affected := changed[key.payer]; reset || affected {
entry.version++
Expand Down
44 changes: 40 additions & 4 deletions eth/payment_observer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,24 @@ func TestObserverRejectedHeadersInvalidateConflictingIdentity(t *testing.T) {
}

func TestObserverL1ProgressDeadline(t *testing.T) {
for _, clocks := range [][]uint64{{50, 50, 50}, {49, 50, 49}} {
elapsed := []time.Duration{10, 20, 30, 40, 50, 59, 60}
for _, tc := range []struct {
requireL1 bool
clocks [7]uint64
}{
{true, [7]uint64{50, 50, 50, 50, 50, 50, 50}},
{false, [7]uint64{49, 50, 49, 50, 49, 50, 49}},
} {
f, o, now := newTestObserver(t)
o.requireL1 = tc.requireL1
require.NoError(t, o.refresh(t.Context()))
progress, accepted := o.Status().L1ProgressAt, o.Status().ObservedAt
for step, clock := range clocks {
*now = now.Add(10 * time.Second)
for step, clock := range tc.clocks {
*now = progress.Add(elapsed[step] * time.Second)
advanceObserver(f, *now)
f.Update(func(s *test.ChainState) { s.L1Block = clock })
err := o.refresh(t.Context())
if step < 2 {
if step < len(tc.clocks)-1 {
require.NoError(t, err)
accepted = o.Status().ObservedAt
} else {
Expand All @@ -196,6 +204,34 @@ func TestObserverL1ProgressDeadline(t *testing.T) {
}
}

func TestObserverNativeProgressDeadline(t *testing.T) {
for _, omitL1 := range []bool{true, false} {
f, o, now := newTestObserver(t)
o.requireL1 = false
f.Update(func(s *test.ChainState) { s.Number, s.OmitL1 = 100, true })
require.NoError(t, o.refresh(t.Context()))

*now = now.Add(29 * time.Second)
f.Update(func(s *test.ChainState) {
s.Hash, s.Timestamp = common.HexToHash("0xbeef"), uint64(now.Unix())
s.L1Block, s.OmitL1 = 50, omitL1
})
require.NoError(t, o.refresh(t.Context()))
require.True(t, o.Status().Healthy)

*now = now.Add(time.Second)
require.ErrorIs(t, o.refresh(t.Context()), ErrObservationUnavailable)
require.Equal(t, "chain_l1_progress_stale", o.Status().RecipientRejection)

if omitL1 {
*now = now.Add(5 * time.Second)
advanceObserver(f, *now)
f.Update(func(s *test.ChainState) { s.L1Block, s.OmitL1 = 50, false })
require.ErrorIs(t, o.refresh(t.Context()), ErrObservationUnavailable)
}
}
}

func TestObserverExpiredCandidateIsNotPublished(t *testing.T) {
for _, phase := range []string{"RoundsManager.currentRound", "eth_getLogs"} {
t.Run(phase, func(t *testing.T) {
Expand Down
Loading