From dd7f26308d731cbe7c2e5f7bb5c02b936b942ccf Mon Sep 17 00:00:00 2001 From: mlsmaycon Date: Tue, 18 Aug 2026 19:31:52 +0000 Subject: [PATCH] [e2e] Retry the endpoint probe when the tunnel is not up yet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ResolveProxyIP exists to wake the lazy proxy peer, and it retried only curl exit 6 — DNS. The wake-up attempt that arrives before WireGuard has brought the tunnel up fails with exit 7 instead, and that returned immediately: no HTTP response from vast-azalea.netbird.local: exit status 7 (curl: (7) Failed to connect ... after 0 ms) So the one function whose job is to tolerate a not-yet-ready endpoint failed on the readiness state it was written for, one second after the client container reported ready. Retry both exit codes within the same window; anything else would still be failing when the window closed and still fails immediately. Raise the access-log ingest window to 60s for the same reason. The proxy streams each entry with a 10s send timeout of its own, so 30s left barely three attempts of headroom before a test that had already got its 200 was failed for a row still in flight. --- e2e/agentnetwork/custom_pricing_test.go | 7 ++++- e2e/harness/client.go | 36 +++++++++++++++++-------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/e2e/agentnetwork/custom_pricing_test.go b/e2e/agentnetwork/custom_pricing_test.go index b3ca5028f..90e198d3d 100644 --- a/e2e/agentnetwork/custom_pricing_test.go +++ b/e2e/agentnetwork/custom_pricing_test.go @@ -174,7 +174,12 @@ func chatOnce(t *testing.T, ctx context.Context, env pricedEnv, model, sessionID // accessLogIngestWindow is how long a single request's access-log row is given // to appear before the caller gives up on it. -const accessLogIngestWindow = 30 * time.Second +// accessLogIngestWindow bounds how long a row may take to appear after its +// request returned. The proxy streams each entry to management with a 10s send +// timeout of its own, so a request whose send hits one full timeout and is +// retried has not yet missed anything real — 30s left barely three send +// attempts of headroom and lost the race on a loaded runner. +const accessLogIngestWindow = 60 * time.Second // accessLogPollInterval is how long the lookup waits between pages. Ingest is // asynchronous, so the row lands somewhere inside the window rather than on diff --git a/e2e/harness/client.go b/e2e/harness/client.go index 9e9e7b34a..73931027d 100644 --- a/e2e/harness/client.go +++ b/e2e/harness/client.go @@ -200,12 +200,18 @@ func (cl *Client) pollStatus(ctx context.Context, timeout time.Duration, want st const ( // curlExitCouldNotResolve is curl's exit code for a DNS resolution failure, distinct from connection-level failures. curlExitCouldNotResolve = 6 - // dnsProbeRetryWindow bounds DNS-failure retries: the synthesized zone lands a beat after management connects, so early NXDOMAIN is propagation; a zone still absent after this window is a real failure. - dnsProbeRetryWindow = 30 * time.Second - dnsProbeRetryInterval = 2 * time.Second + // curlExitCouldNotConnect is curl's exit code for a connection that never + // established. The probe exists to WAKE the lazy proxy peer, so the first + // attempt legitimately arrives before WireGuard has brought the tunnel up + // and fails here — which is propagation, exactly like an early NXDOMAIN, + // and belongs inside the retry window rather than failing the test outright. + curlExitCouldNotConnect = 7 + // endpointProbeRetryWindow bounds retries of the transient failures above: the synthesized zone and the tunnel both land a beat after management connects. Still failing after this window is a real failure. + endpointProbeRetryWindow = 30 * time.Second + endpointProbeRetryInterval = 2 * time.Second ) -// ResolveProxyIP GETs https:/// from the client's netns: any HTTP status proves DNS + tunnel and wakes the lazy proxy peer; only DNS failures retry, within dnsProbeRetryWindow. Returns the connected IP for --resolve pinning. +// ResolveProxyIP GETs https:/// from the client's netns: any HTTP status proves DNS + tunnel and wakes the lazy proxy peer; DNS and connect failures retry, within endpointProbeRetryWindow. Returns the connected IP for --resolve pinning. func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, error) { args := []string{ "run", "--rm", @@ -216,7 +222,7 @@ func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, "-w", "%{remote_ip}", "https://" + endpoint + "/", } - deadline := time.Now().Add(dnsProbeRetryWindow) + deadline := time.Now().Add(endpointProbeRetryWindow) for { cmd := exec.CommandContext(ctx, "docker", args...) var stdout, stderr strings.Builder @@ -232,21 +238,29 @@ func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, } var exitErr *exec.ExitError - if !errors.As(err, &exitErr) || exitErr.ExitCode() != curlExitCouldNotResolve { + if !errors.As(err, &exitErr) || !isTransientProbeExit(exitErr.ExitCode()) { return "", fmt.Errorf("no HTTP response from %s: %w (%s)", endpoint, err, strings.TrimSpace(stderr.String())) } - dnsErr := fmt.Errorf("DNS resolution failed for %s: %s", endpoint, strings.TrimSpace(stderr.String())) - if time.Until(deadline) < dnsProbeRetryInterval { - return "", dnsErr + probeErr := fmt.Errorf("endpoint %s not reachable yet: %s", endpoint, strings.TrimSpace(stderr.String())) + if time.Until(deadline) < endpointProbeRetryInterval { + return "", probeErr } select { case <-ctx.Done(): - return "", fmt.Errorf("%w (%w)", dnsErr, ctx.Err()) - case <-time.After(dnsProbeRetryInterval): + return "", fmt.Errorf("%w (%w)", probeErr, ctx.Err()) + case <-time.After(endpointProbeRetryInterval): } } } +// isTransientProbeExit reports whether a curl exit code describes a state the +// endpoint is expected to pass THROUGH on its way up, rather than a settled +// failure. Anything else — TLS refusal, a protocol error, a bad argument — +// would still be failing after the retry window, so it fails immediately. +func isTransientProbeExit(code int) bool { + return code == curlExitCouldNotResolve || code == curlExitCouldNotConnect +} + // Wire shapes for Chat. const ( // WireChat is the OpenAI-compatible /v1/chat/completions shape.