mirror of
https://github.com/netbirdio/netbird.git
synced 2026-08-24 16:41:30 +02:00
[e2e] Retry the endpoint probe when the tunnel is not up yet
ResolveProxyIP exists to wake the lazy proxy peer, and it retried only curl exit 6 — DNS. The wake-up attempt that arrives before WireGuard has brought the tunnel up fails with exit 7 instead, and that returned immediately: no HTTP response from vast-azalea.netbird.local: exit status 7 (curl: (7) Failed to connect ... after 0 ms) So the one function whose job is to tolerate a not-yet-ready endpoint failed on the readiness state it was written for, one second after the client container reported ready. Retry both exit codes within the same window; anything else would still be failing when the window closed and still fails immediately. Raise the access-log ingest window to 60s for the same reason. The proxy streams each entry with a 10s send timeout of its own, so 30s left barely three attempts of headroom before a test that had already got its 200 was failed for a row still in flight.
This commit is contained in:
@@ -174,7 +174,12 @@ func chatOnce(t *testing.T, ctx context.Context, env pricedEnv, model, sessionID
|
||||
|
||||
// accessLogIngestWindow is how long a single request's access-log row is given
|
||||
// to appear before the caller gives up on it.
|
||||
const accessLogIngestWindow = 30 * time.Second
|
||||
// accessLogIngestWindow bounds how long a row may take to appear after its
|
||||
// request returned. The proxy streams each entry to management with a 10s send
|
||||
// timeout of its own, so a request whose send hits one full timeout and is
|
||||
// retried has not yet missed anything real — 30s left barely three send
|
||||
// attempts of headroom and lost the race on a loaded runner.
|
||||
const accessLogIngestWindow = 60 * time.Second
|
||||
|
||||
// accessLogPollInterval is how long the lookup waits between pages. Ingest is
|
||||
// asynchronous, so the row lands somewhere inside the window rather than on
|
||||
|
||||
@@ -200,12 +200,18 @@ func (cl *Client) pollStatus(ctx context.Context, timeout time.Duration, want st
|
||||
const (
|
||||
// curlExitCouldNotResolve is curl's exit code for a DNS resolution failure, distinct from connection-level failures.
|
||||
curlExitCouldNotResolve = 6
|
||||
// dnsProbeRetryWindow bounds DNS-failure retries: the synthesized zone lands a beat after management connects, so early NXDOMAIN is propagation; a zone still absent after this window is a real failure.
|
||||
dnsProbeRetryWindow = 30 * time.Second
|
||||
dnsProbeRetryInterval = 2 * time.Second
|
||||
// curlExitCouldNotConnect is curl's exit code for a connection that never
|
||||
// established. The probe exists to WAKE the lazy proxy peer, so the first
|
||||
// attempt legitimately arrives before WireGuard has brought the tunnel up
|
||||
// and fails here — which is propagation, exactly like an early NXDOMAIN,
|
||||
// and belongs inside the retry window rather than failing the test outright.
|
||||
curlExitCouldNotConnect = 7
|
||||
// endpointProbeRetryWindow bounds retries of the transient failures above: the synthesized zone and the tunnel both land a beat after management connects. Still failing after this window is a real failure.
|
||||
endpointProbeRetryWindow = 30 * time.Second
|
||||
endpointProbeRetryInterval = 2 * time.Second
|
||||
)
|
||||
|
||||
// ResolveProxyIP GETs https://<endpoint>/ from the client's netns: any HTTP status proves DNS + tunnel and wakes the lazy proxy peer; only DNS failures retry, within dnsProbeRetryWindow. Returns the connected IP for --resolve pinning.
|
||||
// ResolveProxyIP GETs https://<endpoint>/ from the client's netns: any HTTP status proves DNS + tunnel and wakes the lazy proxy peer; DNS and connect failures retry, within endpointProbeRetryWindow. Returns the connected IP for --resolve pinning.
|
||||
func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, error) {
|
||||
args := []string{
|
||||
"run", "--rm",
|
||||
@@ -216,7 +222,7 @@ func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string,
|
||||
"-w", "%{remote_ip}",
|
||||
"https://" + endpoint + "/",
|
||||
}
|
||||
deadline := time.Now().Add(dnsProbeRetryWindow)
|
||||
deadline := time.Now().Add(endpointProbeRetryWindow)
|
||||
for {
|
||||
cmd := exec.CommandContext(ctx, "docker", args...)
|
||||
var stdout, stderr strings.Builder
|
||||
@@ -232,21 +238,29 @@ func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string,
|
||||
}
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) || exitErr.ExitCode() != curlExitCouldNotResolve {
|
||||
if !errors.As(err, &exitErr) || !isTransientProbeExit(exitErr.ExitCode()) {
|
||||
return "", fmt.Errorf("no HTTP response from %s: %w (%s)", endpoint, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
dnsErr := fmt.Errorf("DNS resolution failed for %s: %s", endpoint, strings.TrimSpace(stderr.String()))
|
||||
if time.Until(deadline) < dnsProbeRetryInterval {
|
||||
return "", dnsErr
|
||||
probeErr := fmt.Errorf("endpoint %s not reachable yet: %s", endpoint, strings.TrimSpace(stderr.String()))
|
||||
if time.Until(deadline) < endpointProbeRetryInterval {
|
||||
return "", probeErr
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", fmt.Errorf("%w (%w)", dnsErr, ctx.Err())
|
||||
case <-time.After(dnsProbeRetryInterval):
|
||||
return "", fmt.Errorf("%w (%w)", probeErr, ctx.Err())
|
||||
case <-time.After(endpointProbeRetryInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// isTransientProbeExit reports whether a curl exit code describes a state the
|
||||
// endpoint is expected to pass THROUGH on its way up, rather than a settled
|
||||
// failure. Anything else — TLS refusal, a protocol error, a bad argument —
|
||||
// would still be failing after the retry window, so it fails immediately.
|
||||
func isTransientProbeExit(code int) bool {
|
||||
return code == curlExitCouldNotResolve || code == curlExitCouldNotConnect
|
||||
}
|
||||
|
||||
// Wire shapes for Chat.
|
||||
const (
|
||||
// WireChat is the OpenAI-compatible /v1/chat/completions shape.
|
||||
|
||||
Reference in New Issue
Block a user