Merge pull request #109 from fosrl/dev

1.5.0

.github/dependabot.yml

@@ -1,40 +1,32 @@
 version: 2
+
 updates:
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
       interval: "daily"
+    open-pull-requests-limit: 1
     groups:
-      dev-patch-updates:
-        dependency-type: "development"
-        update-types:
-          - "patch"
-      dev-minor-updates:
-        dependency-type: "development"
-        update-types:
-          - "minor"
-      prod-patch-updates:
-        dependency-type: "production"
-        update-types:
-          - "patch"
-      prod-minor-updates:
-        dependency-type: "production"
-        update-types:
-          - "minor"
+      go-dependencies:
+        patterns:
+          - "*"
 
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
       interval: "daily"
+    open-pull-requests-limit: 1
     groups:
-      patch-updates:
-        update-types:
-          - "patch"
-      minor-updates:
-        update-types:
-          - "minor"
+      docker-dependencies:
+        patterns:
+          - "*"
 
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
+    open-pull-requests-limit: 1
+    groups:
+      github-actions-dependencies:
+        patterns:
+          - "*"

.github/workflows/cicd.yml

@@ -36,16 +36,16 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
         with:
           registry: docker.io
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
@@ -57,7 +57,7 @@ jobs:
         shell: bash
 
       - name: Install Go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
         with:
           go-version: 1.26
 
@@ -80,7 +80,7 @@ jobs:
         shell: bash
 
       - name: Login in to GHCR
-        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -156,7 +156,7 @@ jobs:
         shell: bash
 
       - name: Upload artifacts from /bin
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: binaries
           path: bin/

.github/workflows/test.yml

@@ -14,14 +14,14 @@ jobs:
     runs-on: amd64-runner
 
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Read go version
         id: goversion
         run: echo "version=$(cat .go-version)" >> $GITHUB_OUTPUT
 
       - name: Set up Go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
         with:
           go-version: ${{ steps.goversion.outputs.version }}
 

Dockerfile

@@ -17,7 +17,7 @@ COPY . .
 RUN CGO_ENABLED=0 GOOS=linux go build -o /gerbil
 
 # Start a new stage from scratch
-FROM alpine:3.23 AS runner
+FROM alpine:3.24 AS runner
 
 RUN apk add --no-cache iptables iproute2
 

go.mod

@@ -12,8 +12,8 @@ require (
 	go.opentelemetry.io/otel/metric v1.44.0
 	go.opentelemetry.io/otel/sdk v1.44.0
 	go.opentelemetry.io/otel/sdk/metric v1.44.0
-	golang.org/x/crypto v0.52.0
-	golang.org/x/sync v0.20.0
+	golang.org/x/crypto v0.53.0
+	golang.org/x/sync v0.21.0
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6
 )
 
@@ -40,8 +40,8 @@ require (
 	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/net v0.55.0 // indirect
-	golang.org/x/sys v0.45.0 // indirect
-	golang.org/x/text v0.37.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
+	golang.org/x/text v0.38.0 // indirect
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect

go.sum

@@ -83,18 +83,18 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
-golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
+golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
+golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
 golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
 golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
-golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
-golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
-golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
-golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
+golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 h1:CawjfCvYQH2OU3/TnxLx97WDSUDRABfT18pCOYwc2GE=

internal/metrics/metrics.go

@@ -58,6 +58,7 @@ var (
 	udpPacketSizeBytes         observability.Histogram
 	holePunchEventsTotal       observability.Counter
 	proxyMappingActive         observability.UpDownCounter
+	relayUDPConnectionsActive  observability.UpDownCounter
 	sessionRebuiltTotal        observability.Counter
 	commPatternActive          observability.UpDownCounter
 	proxyCleanupRemovedTotal   observability.Counter
@@ -342,6 +343,11 @@ func createInstruments() error {
 	if err != nil {
 		return err
 	}
+	relayUDPConnectionsActive, err = newUpDownCounter("gerbil_relay_udp_connections_active",
+		"Number of open per-peer outbound UDP sockets held by the relay connection pool", "ifname")
+	if err != nil {
+		return err
+	}
 	sessionRebuiltTotal, err = newCounter("gerbil_session_rebuilt_total",
 		"Count of sessions rebuilt from communication patterns", "ifname")
 	if err != nil {
@@ -711,6 +717,13 @@ func RecordSession(ifname string, delta int64) {
 	activeSessions.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
 }
 
+func RecordUDPConnection(ifname string, delta int64) {
+	if relayUDPConnectionsActive == nil {
+		return
+	}
+	relayUDPConnectionsActive.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
+}
+
 func RecordSessionRebuilt(ifname string) {
 	if sessionRebuiltTotal == nil {
 		return

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -11,10 +12,12 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	_ "net/http/pprof"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"runtime/pprof"
 	"strconv"
@@ -46,6 +49,7 @@ var (
 	doTrafficShaping bool
 	bandwidthLimit   string
 	ifbName          string // IFB device name for ingress traffic shaping
+	disableFirewall  bool
 )
 
 type WgConfig struct {
@@ -102,6 +106,144 @@ type UpdateDestinationsRequest struct {
 	Destinations []relay.PeerDestination `json:"destinations"`
 }
 
+// pangolinDestHeader carries the downstream host:port (reachable over the
+// WireGuard interface) that a /router request should be rewritten to,
+// optionally prefixed with a scheme (e.g. "https://100.96.128.1:443"). It is
+// stripped before the request is forwarded.
+const pangolinDestHeader = "p-dest-header"
+
+// pangolinHostHeader optionally carries the Host header value that should be
+// sent to the destination, when it differs from pangolinDestHeader (e.g. the
+// target's configured IP/hostname rather than the WireGuard routing
+// address). It is stripped before the request is forwarded. When absent,
+// the destination from pangolinDestHeader is used as the Host header, same
+// as before.
+const pangolinHostHeader = "p-dest-host-header"
+
+// splitDestHeader separates an optional "scheme://" prefix from a
+// pangolinDestHeader value, defaulting to "http" when none is present.
+func splitDestHeader(dest string) (scheme, host string) {
+	if s, rest, ok := strings.Cut(dest, "://"); ok {
+		return s, rest
+	}
+	return "http", dest
+}
+
+// hostname strips an optional ":port" suffix from a host header value.
+func hostname(hostport string) string {
+	if h, _, err := net.SplitHostPort(hostport); err == nil {
+		return h
+	}
+	return hostport
+}
+
+// routerSNIContextKey carries the TLS ServerName (SNI) that
+// routerTransport's DialTLSContext should present, since we dial the
+// WireGuard destination IP but the remote end typically terminates TLS
+// based on the original hostname (pangolinHostHeader), not that IP.
+type routerSNIContextKey struct{}
+
+// routerTransport is routerProxy's RoundTripper. It mirrors
+// http.DefaultTransport except for TLS dials, where it sets the SNI from
+// routerSNIContextKey instead of letting it default to the dial address
+// (the WireGuard IP), which the destination's TLS termination won't have a
+// matching certificate/route for.
+var routerTransport = &http.Transport{
+	Proxy: http.ProxyFromEnvironment,
+	DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+		serverName := hostname(addr)
+		if sni, ok := ctx.Value(routerSNIContextKey{}).(string); ok && sni != "" {
+			serverName = sni
+		}
+		dialer := &tls.Dialer{Config: &tls.Config{ServerName: serverName}}
+		return dialer.DialContext(ctx, network, addr)
+	},
+}
+
+// logDebugRequest dumps a request's destination, headers, and body at debug
+// level for troubleshooting (e.g. verifying an auth header made it through
+// the proxy chain intact). It reads and restores req.Body so the request can
+// still be sent afterward. Header values (including secrets like API keys)
+// are logged as-is - only intended to be enabled for local troubleshooting.
+func logDebugRequest(label string, req *http.Request) {
+	var headerLines strings.Builder
+	for name, values := range req.Header {
+		for _, v := range values {
+			fmt.Fprintf(&headerLines, "\n  %s: %s", name, v)
+		}
+	}
+
+	body := []byte("<empty>")
+	if req.Body != nil {
+		data, err := io.ReadAll(req.Body)
+		req.Body.Close()
+		if err != nil {
+			logger.Error("%s: failed to read body for logging: %v", label, err)
+		} else {
+			body = data
+		}
+		req.Body = io.NopCloser(bytes.NewReader(data))
+	}
+
+	logger.Debug("%s: %s %s://%s%s headers:%s\nbody: %s", label, req.Method, req.URL.Scheme, req.Host, req.URL.RequestURI(), headerLines.String(), body)
+}
+
+// routerProxy forwards /router/* requests from the Pangolin AI gateway to a
+// destination on the WireGuard network, as named by pangolinDestHeader.
+// Used for proxying AI chat completion requests (incl. streaming) to
+// providers reachable only from a site.
+var routerProxy = &httputil.ReverseProxy{
+	Rewrite: func(pr *httputil.ProxyRequest) {
+		scheme, dest := splitDestHeader(pr.In.Header.Get(pangolinDestHeader))
+
+		pr.Out.URL.Scheme = scheme
+		pr.Out.URL.Host = dest
+		pr.Out.URL.Path = strings.TrimPrefix(pr.In.URL.Path, "/router")
+		if !strings.HasPrefix(pr.Out.URL.Path, "/") {
+			pr.Out.URL.Path = "/" + pr.Out.URL.Path
+		}
+		pr.Out.URL.RawPath = ""
+		pr.Out.Host = dest
+		if hostOverride := pr.In.Header.Get(pangolinHostHeader); hostOverride != "" {
+			pr.Out.Host = hostOverride
+			ctx := context.WithValue(pr.Out.Context(), routerSNIContextKey{}, hostname(hostOverride))
+			pr.Out = pr.Out.WithContext(ctx)
+		}
+
+		pr.Out.Header.Del(pangolinDestHeader)
+		pr.Out.Header.Del(pangolinHostHeader)
+
+		logger.Debug("Router proxy: %s %s -> %s (Host: %s)", pr.In.Method, pr.In.URL.Path, pr.Out.URL.String(), pr.Out.Host)
+		logDebugRequest("Router outbound request", pr.Out)
+	},
+	Transport: routerTransport,
+	// Flush written bytes to the client immediately rather than buffering,
+	// which is required for SSE-based streaming chat completions.
+	FlushInterval: -1,
+	ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
+		logger.Error("Router proxy error for %s: %v", r.URL.Path, err)
+		http.Error(w, "Bad gateway", http.StatusBadGateway)
+	},
+}
+
+func handleRouter(w http.ResponseWriter, r *http.Request) {
+	dest := r.Header.Get(pangolinDestHeader)
+	hostOverride := r.Header.Get(pangolinHostHeader)
+	logger.Debug("Router request received: %s %s dest=%s host=%s remote=%s", r.Method, r.URL.Path, dest, hostOverride, r.RemoteAddr)
+
+	if dest == "" {
+		http.Error(w, fmt.Sprintf("Missing %s header", pangolinDestHeader), http.StatusBadRequest)
+		return
+	}
+	_, host := splitDestHeader(dest)
+	if _, _, err := net.SplitHostPort(host); err != nil {
+		http.Error(w, "Invalid destination", http.StatusBadRequest)
+		return
+	}
+
+	routerProxy.ServeHTTP(w, r)
+}
+
 // httpMetricsMiddleware wraps HTTP handlers with metrics tracking
 func httpMetricsMiddleware(endpoint string, handler http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -131,6 +273,16 @@ func (w *responseWriterWrapper) WriteHeader(statusCode int) {
 	w.ResponseWriter.WriteHeader(statusCode)
 }
 
+// Unwrap exposes the underlying ResponseWriter so http.ResponseController
+// (used by httputil.ReverseProxy's streaming Flush, and by Hijack/Push
+// callers) can see through this wrapper to the real http.Flusher etc.
+// Without this, ReverseProxy's flushes on /router/* silently no-op and
+// streamed responses (e.g. SSE) get buffered until the response completes
+// instead of being forwarded incrementally.
+func (w *responseWriterWrapper) Unwrap() http.ResponseWriter {
+	return w.ResponseWriter
+}
+
 func parseLogLevel(level string) logger.LogLevel {
 	switch strings.ToUpper(level) {
 	case "DEBUG":
@@ -149,8 +301,6 @@ func parseLogLevel(level string) logger.LogLevel {
 }
 
 func main() {
-	go monitorMemory(1024 * 1024 * 512) // trigger if memory usage exceeds 512MB
-
 	var (
 		err                  error
 		wgconfig             WgConfig
@@ -196,6 +346,7 @@ func main() {
 	proxyProtocolStr := os.Getenv("PROXY_PROTOCOL")
 	doTrafficShapingStr := os.Getenv("DO_TRAFFIC_SHAPING")
 	bandwidthLimitStr := os.Getenv("BANDWIDTH_LIMIT")
+	disableFirewallStr := os.Getenv("DISABLE_FIREWALL")
 
 	// Read metrics env vars (defaults applied by DefaultMetricsConfig; these override defaults).
 	metricsEnabled = true // default
@@ -316,6 +467,13 @@ func main() {
 		flag.BoolVar(&doTrafficShaping, "do-traffic-shaping", false, "Whether to set up traffic shaping rules for peers (requires tc command and root privileges)")
 	}
 
+	if disableFirewallStr != "" {
+		disableFirewall = strings.ToLower(disableFirewallStr) == "true"
+	}
+	if disableFirewallStr == "" {
+		flag.BoolVar(&disableFirewall, "disable-firewall", false, "Disable WireGuard firewall rules to allow all inbound traffic on the interface")
+	}
+
 	if bandwidthLimitStr != "" {
 		bandwidthLimit = bandwidthLimitStr
 	}
@@ -335,6 +493,15 @@ func main() {
 
 	flag.Parse()
 
+	// Heap profiles are dumped into the configured config directory (the same
+	// directory as the config file, or /var/config as a fallback for remote-config
+	// setups) so they land on the volume the operator actually has mounted.
+	heapDir := "/var/config/heap"
+	if configFile != "" {
+		heapDir = filepath.Join(filepath.Dir(configFile), "heap")
+	}
+	go monitorMemory(1024*1024*512, heapDir) // trigger if memory usage exceeds 512MB
+
 	// Derive IFB device name from the WireGuard interface name (Linux limit: 15 chars)
 	ifbName = "ifb_" + interfaceName
 	if len(ifbName) > 15 {
@@ -551,6 +718,7 @@ func main() {
 	http.HandleFunc("/update-destinations", httpMetricsMiddleware("update_destinations", handleUpdateDestinations))
 	http.HandleFunc("/update-local-snis", httpMetricsMiddleware("update_local_snis", handleUpdateLocalSNIs))
 	http.HandleFunc("/healthz", httpMetricsMiddleware("healthz", handleHealthz))
+	http.HandleFunc("/router/", httpMetricsMiddleware("router", handleRouter))
 
 	// Register metrics endpoint only for Prometheus backend.
 	// OTel backend pushes to a collector; no /metrics endpoint needed.
@@ -732,7 +900,9 @@ func ensureWireguardInterface(wgconfig WgConfig) error {
 		logger.Warn("Failed to ensure MSS clamping: %v", err)
 	}
 
-	if err := ensureWireguardFirewall(); err != nil {
+	if disableFirewall {
+		logger.Warn("Firewall disabled: all inbound traffic on %s will be allowed", interfaceName)
+	} else if err := ensureWireguardFirewall(wgconfig.IpAddress); err != nil {
 		logger.Warn("Failed to ensure WireGuard firewall rules: %v", err)
 	}
 
@@ -908,11 +1078,18 @@ func ensureMSSClamping() error {
 	return nil
 }
 
-func ensureWireguardFirewall() error {
+func ensureWireguardFirewall(localIpAddress string) error {
 	// Rules to enforce:
 	// 1. Allow established/related connections (responses to our outbound traffic)
 	// 2. Allow ICMP ping packets
-	// 3. Drop all other inbound traffic from peers
+	// 3. Allow inbound traffic to ports 80/443 on the local IP only (for Traefik)
+	// 4. Drop all other inbound traffic from peers
+
+	// Strip any CIDR suffix so we're left with just the host IP
+	localIp := localIpAddress
+	if ip, _, err := net.ParseCIDR(localIpAddress); err == nil {
+		localIp = ip.String()
+	}
 
 	// Define the rules we want to ensure exist
 	rules := [][]string{
@@ -932,6 +1109,24 @@ func ensureWireguardFirewall() error {
 			"--icmp-type", "8",
 			"-j", "ACCEPT",
 		},
+		// Allow inbound HTTP to the local IP only (for Traefik)
+		{
+			"-A", "INPUT",
+			"-i", interfaceName,
+			"-p", "tcp",
+			"--dport", "80",
+			"-d", localIp,
+			"-j", "ACCEPT",
+		},
+		// Allow inbound HTTPS to the local IP only (for Traefik)
+		{
+			"-A", "INPUT",
+			"-i", interfaceName,
+			"-p", "tcp",
+			"--dport", "443",
+			"-d", localIp,
+			"-j", "ACCEPT",
+		},
 		// Drop all other inbound traffic from WireGuard interface
 		{
 			"-A", "INPUT",
@@ -1493,6 +1688,24 @@ func calculatePeerBandwidth() ([]PeerBandwidth, error) {
 	return peerBandwidths, nil
 }
 
+// defaultBandwidthReportBatchSize caps how many peer bandwidth readings are
+// sent in a single POST. Reporting every peer in one request grows unbounded
+// with fleet size and can exceed the remote server's request body limit,
+// which surfaces as "413 Payload Too Large" and silently drops that whole
+// report cycle. Overridable via GERBIL_BANDWIDTH_BATCH_SIZE.
+const defaultBandwidthReportBatchSize = 250
+
+var bandwidthReportBatchSize = loadBandwidthReportBatchSize()
+
+func loadBandwidthReportBatchSize() int {
+	if v := os.Getenv("GERBIL_BANDWIDTH_BATCH_SIZE"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			return n
+		}
+	}
+	return defaultBandwidthReportBatchSize
+}
+
 func reportPeerBandwidth(apiURL string) error {
 	bandwidths, err := calculatePeerBandwidth()
 	if err != nil {
@@ -1501,26 +1714,47 @@ func reportPeerBandwidth(apiURL string) error {
 		return fmt.Errorf("failed to calculate peer bandwidth: %v", err)
 	}
 
-	jsonData, err := json.Marshal(bandwidths)
+	if len(bandwidths) == 0 {
+		return nil
+	}
+
+	var batchErrs []error
+	for start := 0; start < len(bandwidths); start += bandwidthReportBatchSize {
+		end := start + bandwidthReportBatchSize
+		if end > len(bandwidths) {
+			end = len(bandwidths)
+		}
+
+		if err := sendPeerBandwidthBatch(apiURL, bandwidths[start:end]); err != nil {
+			metrics.RecordBandwidthReport("error")
+			batchErrs = append(batchErrs, err)
+			continue
+		}
+		metrics.RecordBandwidthReport("success")
+	}
+
+	if len(batchErrs) > 0 {
+		return fmt.Errorf("failed to report %d bandwidth batch(es): %w", len(batchErrs), errors.Join(batchErrs...))
+	}
+	return nil
+}
+
+func sendPeerBandwidthBatch(apiURL string, batch []PeerBandwidth) error {
+	jsonData, err := json.Marshal(batch)
 	if err != nil {
-		metrics.RecordBandwidthReport("error")
 		return fmt.Errorf("failed to marshal bandwidth data: %v", err)
 	}
 
 	resp, err := http.Post(apiURL, "application/json", bytes.NewBuffer(jsonData))
 	if err != nil {
-		metrics.RecordBandwidthReport("error")
 		return fmt.Errorf("failed to send bandwidth data: %v", err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		metrics.RecordBandwidthReport("error")
 		return fmt.Errorf("API returned non-OK status: %s", resp.Status)
 	}
 
-	// Record successful bandwidth report
-	metrics.RecordBandwidthReport("success")
 	return nil
 }
 
@@ -1549,7 +1783,7 @@ func notifyPeerChange(action, publicKey string) {
 	}
 }
 
-func monitorMemory(limit uint64) {
+func monitorMemory(limit uint64, heapDir string) {
 	var m runtime.MemStats
 	for {
 		runtime.ReadMemStats(&m)
@@ -1565,8 +1799,9 @@ func monitorMemory(limit uint64) {
 			// Record memory spike metric
 			metrics.RecordMemorySpike(severity)
 
-			f, err := os.Create(fmt.Sprintf("/var/config/heap/heap-spike-%d.pprof", time.Now().Unix()))
-			if err != nil {
+			if err := os.MkdirAll(heapDir, 0o755); err != nil {
+				log.Println("could not create heap profile directory:", err)
+			} else if f, err := os.Create(filepath.Join(heapDir, fmt.Sprintf("heap-spike-%d.pprof", time.Now().Unix()))); err != nil {
 				log.Println("could not create profile:", err)
 			} else {
 				pprof.WriteHeapProfile(f)

relay/relay.go

@@ -10,8 +10,11 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"os"
 	"runtime"
+	"strconv"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/fosrl/gerbil/internal/metrics"
@@ -60,8 +63,31 @@ type PeerDestination struct {
 }
 
 type DestinationConn struct {
-	conn     *net.UDPConn
-	lastUsed time.Time
+	conn *net.UDPConn
+	// lastUsed is unix nanoseconds, read/written via atomic ops since it's
+	// touched from packet workers and the response goroutine concurrently
+	// with no lock, and is also scanned for LRU eviction below.
+	lastUsed atomic.Int64
+}
+
+// defaultMaxUDPConnections caps the number of concurrent per-peer outbound
+// UDP sockets the relay will keep open in s.connections. Without a cap, a
+// burst of peer churn creates sockets faster than the 5-minute idle cleanup
+// can reap them, exhausting the host's ephemeral port range or fd ulimit.
+// That surfaces as "dial udp ...: resource temporarily unavailable" on every
+// subsequent packet and pegs the CPU logging the flood (outage 2026-07-03,
+// recurrence 2026-07-05). Overridable via GERBIL_MAX_UDP_CONNECTIONS.
+const defaultMaxUDPConnections = 8192
+
+var maxUDPConnections = loadMaxUDPConnections()
+
+func loadMaxUDPConnections() int64 {
+	if v := os.Getenv("GERBIL_MAX_UDP_CONNECTIONS"); v != "" {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
+			return n
+		}
+	}
+	return defaultMaxUDPConnections
 }
 
 // Type for storing WireGuard handshake information
@@ -149,6 +175,13 @@ type cachedEndpointState struct {
 	PublicKey string
 }
 
+// cachedEndpointEntry wraps cachedEndpointState with a timestamp so the cache
+// can be expired after a short TTL even when the endpoint fields are unchanged.
+type cachedEndpointEntry struct {
+	state    cachedEndpointState
+	cachedAt time.Time
+}
+
 // --- End Types ---
 
 // bufferPool allows reusing buffers to reduce allocations.
@@ -165,10 +198,13 @@ type UDPProxyServer struct {
 	conn          *net.UDPConn
 	proxyMappings sync.Map // map[string]ProxyMapping where key is "ip:port"
 	connections   sync.Map // map[string]*DestinationConn where key is destination "ip:port"
-	privateKey    wgtypes.Key
-	packetChan    chan Packet
-	ctx           context.Context
-	cancel        context.CancelFunc
+	// connectionCount mirrors len(connections) without an O(n) sync.Map walk,
+	// so the cap check in getOrCreateConnection is cheap on the hot path.
+	connectionCount atomic.Int64
+	privateKey      wgtypes.Key
+	packetChan      chan Packet
+	ctx             context.Context
+	cancel          context.CancelFunc
 
 	// Session tracking for WireGuard peers
 	// Key format: "senderIndex:receiverIndex"
@@ -406,6 +442,9 @@ func (s *UDPProxyServer) packetWorker() {
 			logger.Debug("Created endpoint from packet remoteAddr %s: IP=%s, Port=%d", packet.remoteAddr.String(), endpoint.IP, endpoint.Port)
 
 			// Check if anything meaningful changed before queuing an HTTP notification.
+			// The cache expires after 2.5 s so the server always receives a fresh
+			// timestamp within its 5-second staleness window.
+			const endpointCacheTTL = 2500 * time.Millisecond
 			cacheKey := endpoint.OlmID + ":" + endpoint.NewtID
 			newState := cachedEndpointState{
 				OlmID:     endpoint.OlmID,
@@ -415,16 +454,19 @@ func (s *UDPProxyServer) packetWorker() {
 				Port:      endpoint.Port,
 				PublicKey: endpoint.ClientPublicKey,
 			}
-			if cached, ok := s.lastEndpointCache.Load(cacheKey); ok && cached.(cachedEndpointState) == newState {
-				// Endpoint unchanged - skip the HTTP call but still clear stale sessions.
-				logger.Debug("Endpoint unchanged for %s, skipping notification", cacheKey)
-				metrics.RecordHolePunchEvent(relayIfname, "deduplicated")
-				s.clearSessionsForIP(endpoint.IP)
-				metrics.RecordHolePunchEvent(relayIfname, "success")
-				bufferPool.Put(packet.data[:1500])
-				continue
+			if cached, ok := s.lastEndpointCache.Load(cacheKey); ok {
+				entry := cached.(cachedEndpointEntry)
+				if entry.state == newState && time.Since(entry.cachedAt) < endpointCacheTTL {
+					// Endpoint unchanged and cache still fresh - skip the HTTP call.
+					logger.Debug("Endpoint unchanged for %s, skipping notification", cacheKey)
+					metrics.RecordHolePunchEvent(relayIfname, "deduplicated")
+					s.clearSessionsForIP(endpoint.IP)
+					metrics.RecordHolePunchEvent(relayIfname, "success")
+					bufferPool.Put(packet.data[:1500])
+					continue
+				}
 			}
-			s.lastEndpointCache.Store(cacheKey, newState)
+			s.lastEndpointCache.Store(cacheKey, cachedEndpointEntry{state: newState, cachedAt: time.Now()})
 
 			// Queue the notification asynchronously so the hot path is not blocked by HTTP.
 			select {
@@ -849,10 +891,17 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
 	// Check if we have an existing connection
 	if conn, ok := s.connections.Load(key); ok {
 		destConn := conn.(*DestinationConn)
-		destConn.lastUsed = time.Now()
+		destConn.lastUsed.Store(time.Now().UnixNano())
 		return destConn.conn, nil
 	}
 
+	// Enforce a hard cap on concurrent sockets so a burst of peer churn can't
+	// exhaust the host's ephemeral ports/fds. Evict the least-recently-used
+	// connection to make room instead of growing unbounded.
+	if s.connectionCount.Load() >= maxUDPConnections {
+		s.evictLRUConnection()
+	}
+
 	// Create new connection
 	newConn, err := net.DialUDP("udp", nil, destAddr)
 	if err != nil {
@@ -860,11 +909,17 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
 		return nil, fmt.Errorf("failed to create UDP connection: %v", err)
 	}
 
-	// Store the new connection
-	s.connections.Store(key, &DestinationConn{
-		conn:     newConn,
-		lastUsed: time.Now(),
-	})
+	destConn := &DestinationConn{conn: newConn}
+	destConn.lastUsed.Store(time.Now().UnixNano())
+
+	// Store the new connection. If another goroutine raced us and already
+	// created one for this key, close ours and use theirs instead.
+	if existing, loaded := s.connections.LoadOrStore(key, destConn); loaded {
+		newConn.Close()
+		return existing.(*DestinationConn).conn, nil
+	}
+	s.connectionCount.Add(1)
+	metrics.RecordUDPConnection(relayIfname, 1)
 
 	// Start a goroutine to handle responses
 	go s.handleResponses(newConn, destAddr, remoteAddr)
@@ -872,6 +927,33 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
 	return newConn, nil
 }
 
+// evictLRUConnection closes and removes the least-recently-used destination
+// connection so a new one can be created under the concurrent connection cap.
+func (s *UDPProxyServer) evictLRUConnection() {
+	var oldestKey interface{}
+	var oldestConn *DestinationConn
+	var oldestTime int64
+
+	s.connections.Range(func(key, value interface{}) bool {
+		destConn := value.(*DestinationConn)
+		lu := destConn.lastUsed.Load()
+		if oldestKey == nil || lu < oldestTime {
+			oldestKey = key
+			oldestConn = destConn
+			oldestTime = lu
+		}
+		return true
+	})
+
+	if oldestKey != nil {
+		s.connections.Delete(oldestKey)
+		oldestConn.conn.Close()
+		s.connectionCount.Add(-1)
+		metrics.RecordUDPConnection(relayIfname, -1)
+		metrics.RecordProxyCleanupRemoved(relayIfname, "conn_evicted", 1)
+	}
+}
+
 func (s *UDPProxyServer) handleResponses(conn *net.UDPConn, destAddr *net.UDPAddr, remoteAddr *net.UDPAddr) {
 	buffer := make([]byte, 1500)
 	for {
@@ -923,22 +1005,32 @@ func (s *UDPProxyServer) handleResponses(conn *net.UDPConn, destAddr *net.UDPAdd
 
 // Add a cleanup method to periodically remove idle connections
 func (s *UDPProxyServer) cleanupIdleConnections() {
-	ticker := time.NewTicker(5 * time.Minute)
+	// Ticker interval and idle threshold were previously 5min/10min, meaning
+	// a socket could sit open for up to 15 minutes after going idle. Under a
+	// reconnect/churn burst that lag is enough to exhaust ephemeral ports
+	// before cleanup catches up, so both are tightened here.
+	ticker := time.NewTicker(1 * time.Minute)
 	defer ticker.Stop()
 	for {
 		select {
 		case <-ticker.C:
 			cleanupStart := time.Now()
-			now := time.Now()
+			now := time.Now().UnixNano()
+			removed := int64(0)
 			s.connections.Range(func(key, value interface{}) bool {
 				destConn := value.(*DestinationConn)
-				if now.Sub(destConn.lastUsed) > 10*time.Minute {
+				if now-destConn.lastUsed.Load() > int64(5*time.Minute) {
 					destConn.conn.Close()
 					s.connections.Delete(key)
-					metrics.RecordProxyCleanupRemoved(relayIfname, "conn", 1)
+					removed++
 				}
 				return true
 			})
+			if removed > 0 {
+				s.connectionCount.Add(-removed)
+				metrics.RecordUDPConnection(relayIfname, -removed)
+				metrics.RecordProxyCleanupRemoved(relayIfname, "conn", removed)
+			}
 			metrics.RecordProxyIdleCleanupDuration(relayIfname, "conn", time.Since(cleanupStart).Seconds())
 		case <-s.ctx.Done():
 			return
@@ -1100,6 +1192,10 @@ func (s *UDPProxyServer) clearConnectionsForWGIP(wgIP string) {
 	for _, key := range keysToDelete {
 		s.connections.Delete(key)
 	}
+	if len(keysToDelete) > 0 {
+		s.connectionCount.Add(-int64(len(keysToDelete)))
+		metrics.RecordUDPConnection(relayIfname, -int64(len(keysToDelete)))
+	}
 
 	logger.Info("Cleared %d connections for WG IP: %s", len(keysToDelete), wgIP)
 }