[client, management] Recover from stuck proof collections and keep re-tracked accounts on the challenge refresher (#7890)

Co-authored-by: Viktor Liu <viktor@netbird.io>

shared/management/certposture/certposture_test.go

@@ -45,7 +45,9 @@ func TestChallenger_VerifyNonce(t *testing.T) {
 		{"current window", nonce, peerKey, now, nil},
 		{"previous window still accepted", nonce, peerKey, now.Add(Window), nil},
 		{"two windows later expired", nonce, peerKey, now.Add(2 * Window), ErrNonceExpired},
-		{"issued in the future rejected", c.Nonce(peerKey, now.Add(Window)), peerKey, now, ErrNonceExpired},
+		// An instance whose clock runs a little ahead issues the next window's nonce early.
+		{"issued one window ahead accepted", c.Nonce(peerKey, now.Add(Window)), peerKey, now, nil},
+		{"issued two windows ahead rejected", c.Nonce(peerKey, now.Add(2*Window)), peerKey, now, ErrNonceExpired},
 		{"other peer", nonce, otherKey, now, ErrNonceMismatch},
 		{"tampered mac", tamper(nonce, len(nonce)-1), peerKey, now, ErrNonceMismatch},
 		{"malformed", nonce[:10], peerKey, now, ErrNonceMalformed},
@@ -67,7 +69,8 @@ func TestNonceAcceptedAlongside(t *testing.T) {
 	assert.True(t, NonceAcceptedAlongside(issued, issued), "the current nonce is accepted")
 	assert.True(t, NonceAcceptedAlongside(issued, c.Nonce(peerKey, now.Add(Window))), "a proof from the previous window is accepted")
 	assert.False(t, NonceAcceptedAlongside(issued, c.Nonce(peerKey, now.Add(2*Window))), "a proof two windows old is not")
-	assert.False(t, NonceAcceptedAlongside(c.Nonce(peerKey, now.Add(Window)), issued), "a nonce from a later window is not")
+	assert.True(t, NonceAcceptedAlongside(c.Nonce(peerKey, now.Add(Window)), issued), "a nonce from an instance one window ahead is accepted")
+	assert.False(t, NonceAcceptedAlongside(c.Nonce(peerKey, now.Add(2*Window)), issued), "a nonce two windows ahead is not")
 	assert.False(t, NonceAcceptedAlongside([]byte("short"), issued), "a malformed nonce is not")
 
 	for _, tt := range []struct {

shared/management/certposture/challenge.go

@@ -13,9 +13,9 @@ import (
 )
 
 const (
-	// Window is the default challenge window. A nonce is accepted for its own window
-	// and the one before it, so a peer re-proves possession of its key between once
-	// and twice per window.
+	// Window is the default challenge window. A nonce stays valid through the window
+	// after the one it was issued in, so a peer re-proves possession of its key between
+	// once and twice per window. One window early is accepted too, for clock skew.
 	Window = 12 * time.Hour
 
 	// EnvWindow overrides Window, for end-to-end tests that cannot wait half a day to
@@ -23,7 +23,9 @@ const (
 	// window is part of the nonce, so instances that disagree reject each other's.
 	EnvWindow = "NB_CERT_CHALLENGE_WINDOW"
 
-	minWindow = time.Second
+	// minWindow keeps the renewal period, a third of the window, well above the
+	// refresher's one-second tick.
+	minWindow = 30 * time.Second
 	maxWindow = 24 * time.Hour
 
 	challengeDomain = "netbird-cert-challenge-v1"
@@ -58,6 +60,10 @@ func resolveWindow() time.Duration {
 		log.Warnf("%s of %s is outside %s..%s, keeping the %s certificate challenge window", EnvWindow, window, minWindow, maxWindow, Window)
 		return Window
 	}
+	if window%time.Second != 0 {
+		log.Warnf("%s of %s is not a whole number of seconds, keeping the %s certificate challenge window", EnvWindow, window, Window)
+		return Window
+	}
 
 	// Loud on purpose: this sets how long a device can pass the certificate check after
 	// its key has gone, and it has to match on every instance.
@@ -91,8 +97,7 @@ func (c *Challenger) verifyNonce(nonce, peerKey []byte, now time.Time) error {
 		return ErrNonceMalformed
 	}
 	window := binary.BigEndian.Uint64(nonce[:windowLen])
-	current := c.windowOf(now)
-	if window != current && window+1 != current {
+	if !windowAccepted(window, c.windowOf(now)) {
 		return ErrNonceExpired
 	}
 	if !hmac.Equal(nonce, c.nonceForWindow(peerKey, window)) {
@@ -117,13 +122,18 @@ func (c *Challenger) nonceForWindow(peerKey []byte, window uint64) []byte {
 }
 
 // NonceAcceptedAlongside reports whether a proof answering nonce is still accepted while
-// management issues current to the same peer: verification takes a nonce of the current
-// or the previous window.
+// management issues current to the same peer, by the same window rule verification uses.
 func NonceAcceptedAlongside(nonce, current []byte) bool {
 	if len(nonce) != nonceLen || len(current) != nonceLen {
 		return false
 	}
 	window := binary.BigEndian.Uint64(nonce[:windowLen])
-	currentWindow := binary.BigEndian.Uint64(current[:windowLen])
-	return window == currentWindow || window+1 == currentWindow
+	return windowAccepted(window, binary.BigEndian.Uint64(current[:windowLen]))
+}
+
+// windowAccepted reports whether a nonce of window is accepted in window current: the
+// current window, the previous one so a nonce outlives a rollover, and the next one so an
+// instance whose clock runs slightly ahead is not rejected by the others.
+func windowAccepted(window, current uint64) bool {
+	return window == current || window+1 == current || window == current+1
 }

shared/management/certposture/challenge_test.go

@@ -22,6 +22,8 @@ func TestResolveWindow(t *testing.T) {
 		{name: "a test-sized window is taken", env: "30s", want: 30 * time.Second},
 		{name: "garbage keeps the default", env: "soon", want: Window},
 		{name: "below the floor keeps the default", env: "10ms", want: Window},
+		{name: "too short for the refresh tick keeps the default", env: "1s", want: Window},
+		{name: "a fractional second keeps the default", env: "30500ms", want: Window},
 		{name: "above the ceiling keeps the default", env: "100h", want: Window},
 	}