mirror of
https://github.com/fosrl/gerbil.git
synced 2026-09-07 22:51:28 +02:00
Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e25e5766df | ||
|
|
2264c1c7b4 | ||
|
|
904199121d | ||
|
|
5b92bdeb1c | ||
|
|
daecacbd8e | ||
|
|
8efce78d94 | ||
|
|
c29e6ec535 | ||
|
|
0a1496e8de | ||
|
|
4f325dd92f | ||
|
|
8328a4751a | ||
|
|
860ba69a88 | ||
|
|
e1c9836dc1 | ||
|
|
4f457db7b5 | ||
|
|
6d548ea166 | ||
|
|
d611eb2022 |
36
.github/dependabot.yml
vendored
36
.github/dependabot.yml
vendored
@@ -1,40 +1,32 @@
|
||||
version: 2
|
||||
|
||||
updates:
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
open-pull-requests-limit: 1
|
||||
groups:
|
||||
dev-patch-updates:
|
||||
dependency-type: "development"
|
||||
update-types:
|
||||
- "patch"
|
||||
dev-minor-updates:
|
||||
dependency-type: "development"
|
||||
update-types:
|
||||
- "minor"
|
||||
prod-patch-updates:
|
||||
dependency-type: "production"
|
||||
update-types:
|
||||
- "patch"
|
||||
prod-minor-updates:
|
||||
dependency-type: "production"
|
||||
update-types:
|
||||
- "minor"
|
||||
go-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
open-pull-requests-limit: 1
|
||||
groups:
|
||||
patch-updates:
|
||||
update-types:
|
||||
- "patch"
|
||||
minor-updates:
|
||||
update-types:
|
||||
- "minor"
|
||||
docker-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 1
|
||||
groups:
|
||||
github-actions-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
14
.github/workflows/cicd.yml
vendored
14
.github/workflows/cicd.yml
vendored
@@ -36,16 +36,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: docker.io
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
with:
|
||||
go-version: 1.26
|
||||
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
- name: Login in to GHCR
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -156,7 +156,7 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
- name: Upload artifacts from /bin
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: binaries
|
||||
path: bin/
|
||||
|
||||
4
.github/workflows/test.yml
vendored
4
.github/workflows/test.yml
vendored
@@ -14,14 +14,14 @@ jobs:
|
||||
runs-on: amd64-runner
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Read go version
|
||||
id: goversion
|
||||
run: echo "version=$(cat .go-version)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
with:
|
||||
go-version: ${{ steps.goversion.outputs.version }}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /gerbil
|
||||
|
||||
# Start a new stage from scratch
|
||||
FROM alpine:3.23 AS runner
|
||||
FROM alpine:3.24 AS runner
|
||||
|
||||
RUN apk add --no-cache iptables iproute2
|
||||
|
||||
|
||||
8
go.mod
8
go.mod
@@ -12,8 +12,8 @@ require (
|
||||
go.opentelemetry.io/otel/metric v1.44.0
|
||||
go.opentelemetry.io/otel/sdk v1.44.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0
|
||||
golang.org/x/crypto v0.52.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/crypto v0.53.0
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6
|
||||
)
|
||||
|
||||
@@ -40,8 +40,8 @@ require (
|
||||
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||
|
||||
16
go.sum
16
go.sum
@@ -83,18 +83,18 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
|
||||
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
|
||||
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 h1:CawjfCvYQH2OU3/TnxLx97WDSUDRABfT18pCOYwc2GE=
|
||||
|
||||
@@ -58,6 +58,7 @@ var (
|
||||
udpPacketSizeBytes observability.Histogram
|
||||
holePunchEventsTotal observability.Counter
|
||||
proxyMappingActive observability.UpDownCounter
|
||||
relayUDPConnectionsActive observability.UpDownCounter
|
||||
sessionRebuiltTotal observability.Counter
|
||||
commPatternActive observability.UpDownCounter
|
||||
proxyCleanupRemovedTotal observability.Counter
|
||||
@@ -342,6 +343,11 @@ func createInstruments() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relayUDPConnectionsActive, err = newUpDownCounter("gerbil_relay_udp_connections_active",
|
||||
"Number of open per-peer outbound UDP sockets held by the relay connection pool", "ifname")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sessionRebuiltTotal, err = newCounter("gerbil_session_rebuilt_total",
|
||||
"Count of sessions rebuilt from communication patterns", "ifname")
|
||||
if err != nil {
|
||||
@@ -711,6 +717,13 @@ func RecordSession(ifname string, delta int64) {
|
||||
activeSessions.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
|
||||
}
|
||||
|
||||
func RecordUDPConnection(ifname string, delta int64) {
|
||||
if relayUDPConnectionsActive == nil {
|
||||
return
|
||||
}
|
||||
relayUDPConnectionsActive.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
|
||||
}
|
||||
|
||||
func RecordSessionRebuilt(ifname string) {
|
||||
if sessionRebuiltTotal == nil {
|
||||
return
|
||||
|
||||
83
main.go
83
main.go
@@ -15,6 +15,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"strconv"
|
||||
@@ -46,6 +47,7 @@ var (
|
||||
doTrafficShaping bool
|
||||
bandwidthLimit string
|
||||
ifbName string // IFB device name for ingress traffic shaping
|
||||
disableFirewall bool
|
||||
)
|
||||
|
||||
type WgConfig struct {
|
||||
@@ -149,8 +151,6 @@ func parseLogLevel(level string) logger.LogLevel {
|
||||
}
|
||||
|
||||
func main() {
|
||||
go monitorMemory(1024 * 1024 * 512) // trigger if memory usage exceeds 512MB
|
||||
|
||||
var (
|
||||
err error
|
||||
wgconfig WgConfig
|
||||
@@ -196,6 +196,7 @@ func main() {
|
||||
proxyProtocolStr := os.Getenv("PROXY_PROTOCOL")
|
||||
doTrafficShapingStr := os.Getenv("DO_TRAFFIC_SHAPING")
|
||||
bandwidthLimitStr := os.Getenv("BANDWIDTH_LIMIT")
|
||||
disableFirewallStr := os.Getenv("DISABLE_FIREWALL")
|
||||
|
||||
// Read metrics env vars (defaults applied by DefaultMetricsConfig; these override defaults).
|
||||
metricsEnabled = true // default
|
||||
@@ -316,6 +317,13 @@ func main() {
|
||||
flag.BoolVar(&doTrafficShaping, "do-traffic-shaping", false, "Whether to set up traffic shaping rules for peers (requires tc command and root privileges)")
|
||||
}
|
||||
|
||||
if disableFirewallStr != "" {
|
||||
disableFirewall = strings.ToLower(disableFirewallStr) == "true"
|
||||
}
|
||||
if disableFirewallStr == "" {
|
||||
flag.BoolVar(&disableFirewall, "disable-firewall", false, "Disable WireGuard firewall rules to allow all inbound traffic on the interface")
|
||||
}
|
||||
|
||||
if bandwidthLimitStr != "" {
|
||||
bandwidthLimit = bandwidthLimitStr
|
||||
}
|
||||
@@ -335,6 +343,15 @@ func main() {
|
||||
|
||||
flag.Parse()
|
||||
|
||||
// Heap profiles are dumped into the configured config directory (the same
|
||||
// directory as the config file, or /var/config as a fallback for remote-config
|
||||
// setups) so they land on the volume the operator actually has mounted.
|
||||
heapDir := "/var/config/heap"
|
||||
if configFile != "" {
|
||||
heapDir = filepath.Join(filepath.Dir(configFile), "heap")
|
||||
}
|
||||
go monitorMemory(1024*1024*512, heapDir) // trigger if memory usage exceeds 512MB
|
||||
|
||||
// Derive IFB device name from the WireGuard interface name (Linux limit: 15 chars)
|
||||
ifbName = "ifb_" + interfaceName
|
||||
if len(ifbName) > 15 {
|
||||
@@ -732,7 +749,9 @@ func ensureWireguardInterface(wgconfig WgConfig) error {
|
||||
logger.Warn("Failed to ensure MSS clamping: %v", err)
|
||||
}
|
||||
|
||||
if err := ensureWireguardFirewall(); err != nil {
|
||||
if disableFirewall {
|
||||
logger.Warn("Firewall disabled: all inbound traffic on %s will be allowed", interfaceName)
|
||||
} else if err := ensureWireguardFirewall(); err != nil {
|
||||
logger.Warn("Failed to ensure WireGuard firewall rules: %v", err)
|
||||
}
|
||||
|
||||
@@ -1493,6 +1512,24 @@ func calculatePeerBandwidth() ([]PeerBandwidth, error) {
|
||||
return peerBandwidths, nil
|
||||
}
|
||||
|
||||
// defaultBandwidthReportBatchSize caps how many peer bandwidth readings are
|
||||
// sent in a single POST. Reporting every peer in one request grows unbounded
|
||||
// with fleet size and can exceed the remote server's request body limit,
|
||||
// which surfaces as "413 Payload Too Large" and silently drops that whole
|
||||
// report cycle. Overridable via GERBIL_BANDWIDTH_BATCH_SIZE.
|
||||
const defaultBandwidthReportBatchSize = 250
|
||||
|
||||
var bandwidthReportBatchSize = loadBandwidthReportBatchSize()
|
||||
|
||||
func loadBandwidthReportBatchSize() int {
|
||||
if v := os.Getenv("GERBIL_BANDWIDTH_BATCH_SIZE"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return defaultBandwidthReportBatchSize
|
||||
}
|
||||
|
||||
func reportPeerBandwidth(apiURL string) error {
|
||||
bandwidths, err := calculatePeerBandwidth()
|
||||
if err != nil {
|
||||
@@ -1501,26 +1538,47 @@ func reportPeerBandwidth(apiURL string) error {
|
||||
return fmt.Errorf("failed to calculate peer bandwidth: %v", err)
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(bandwidths)
|
||||
if len(bandwidths) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var batchErrs []error
|
||||
for start := 0; start < len(bandwidths); start += bandwidthReportBatchSize {
|
||||
end := start + bandwidthReportBatchSize
|
||||
if end > len(bandwidths) {
|
||||
end = len(bandwidths)
|
||||
}
|
||||
|
||||
if err := sendPeerBandwidthBatch(apiURL, bandwidths[start:end]); err != nil {
|
||||
metrics.RecordBandwidthReport("error")
|
||||
batchErrs = append(batchErrs, err)
|
||||
continue
|
||||
}
|
||||
metrics.RecordBandwidthReport("success")
|
||||
}
|
||||
|
||||
if len(batchErrs) > 0 {
|
||||
return fmt.Errorf("failed to report %d bandwidth batch(es): %w", len(batchErrs), errors.Join(batchErrs...))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sendPeerBandwidthBatch(apiURL string, batch []PeerBandwidth) error {
|
||||
jsonData, err := json.Marshal(batch)
|
||||
if err != nil {
|
||||
metrics.RecordBandwidthReport("error")
|
||||
return fmt.Errorf("failed to marshal bandwidth data: %v", err)
|
||||
}
|
||||
|
||||
resp, err := http.Post(apiURL, "application/json", bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
metrics.RecordBandwidthReport("error")
|
||||
return fmt.Errorf("failed to send bandwidth data: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
metrics.RecordBandwidthReport("error")
|
||||
return fmt.Errorf("API returned non-OK status: %s", resp.Status)
|
||||
}
|
||||
|
||||
// Record successful bandwidth report
|
||||
metrics.RecordBandwidthReport("success")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1549,7 +1607,7 @@ func notifyPeerChange(action, publicKey string) {
|
||||
}
|
||||
}
|
||||
|
||||
func monitorMemory(limit uint64) {
|
||||
func monitorMemory(limit uint64, heapDir string) {
|
||||
var m runtime.MemStats
|
||||
for {
|
||||
runtime.ReadMemStats(&m)
|
||||
@@ -1565,8 +1623,9 @@ func monitorMemory(limit uint64) {
|
||||
// Record memory spike metric
|
||||
metrics.RecordMemorySpike(severity)
|
||||
|
||||
f, err := os.Create(fmt.Sprintf("/var/config/heap/heap-spike-%d.pprof", time.Now().Unix()))
|
||||
if err != nil {
|
||||
if err := os.MkdirAll(heapDir, 0o755); err != nil {
|
||||
log.Println("could not create heap profile directory:", err)
|
||||
} else if f, err := os.Create(filepath.Join(heapDir, fmt.Sprintf("heap-spike-%d.pprof", time.Now().Unix()))); err != nil {
|
||||
log.Println("could not create profile:", err)
|
||||
} else {
|
||||
pprof.WriteHeapProfile(f)
|
||||
|
||||
146
relay/relay.go
146
relay/relay.go
@@ -10,8 +10,11 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/fosrl/gerbil/internal/metrics"
|
||||
@@ -60,8 +63,31 @@ type PeerDestination struct {
|
||||
}
|
||||
|
||||
type DestinationConn struct {
|
||||
conn *net.UDPConn
|
||||
lastUsed time.Time
|
||||
conn *net.UDPConn
|
||||
// lastUsed is unix nanoseconds, read/written via atomic ops since it's
|
||||
// touched from packet workers and the response goroutine concurrently
|
||||
// with no lock, and is also scanned for LRU eviction below.
|
||||
lastUsed atomic.Int64
|
||||
}
|
||||
|
||||
// defaultMaxUDPConnections caps the number of concurrent per-peer outbound
|
||||
// UDP sockets the relay will keep open in s.connections. Without a cap, a
|
||||
// burst of peer churn creates sockets faster than the 5-minute idle cleanup
|
||||
// can reap them, exhausting the host's ephemeral port range or fd ulimit.
|
||||
// That surfaces as "dial udp ...: resource temporarily unavailable" on every
|
||||
// subsequent packet and pegs the CPU logging the flood (outage 2026-07-03,
|
||||
// recurrence 2026-07-05). Overridable via GERBIL_MAX_UDP_CONNECTIONS.
|
||||
const defaultMaxUDPConnections = 8192
|
||||
|
||||
var maxUDPConnections = loadMaxUDPConnections()
|
||||
|
||||
func loadMaxUDPConnections() int64 {
|
||||
if v := os.Getenv("GERBIL_MAX_UDP_CONNECTIONS"); v != "" {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return defaultMaxUDPConnections
|
||||
}
|
||||
|
||||
// Type for storing WireGuard handshake information
|
||||
@@ -149,6 +175,13 @@ type cachedEndpointState struct {
|
||||
PublicKey string
|
||||
}
|
||||
|
||||
// cachedEndpointEntry wraps cachedEndpointState with a timestamp so the cache
|
||||
// can be expired after a short TTL even when the endpoint fields are unchanged.
|
||||
type cachedEndpointEntry struct {
|
||||
state cachedEndpointState
|
||||
cachedAt time.Time
|
||||
}
|
||||
|
||||
// --- End Types ---
|
||||
|
||||
// bufferPool allows reusing buffers to reduce allocations.
|
||||
@@ -165,10 +198,13 @@ type UDPProxyServer struct {
|
||||
conn *net.UDPConn
|
||||
proxyMappings sync.Map // map[string]ProxyMapping where key is "ip:port"
|
||||
connections sync.Map // map[string]*DestinationConn where key is destination "ip:port"
|
||||
privateKey wgtypes.Key
|
||||
packetChan chan Packet
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
// connectionCount mirrors len(connections) without an O(n) sync.Map walk,
|
||||
// so the cap check in getOrCreateConnection is cheap on the hot path.
|
||||
connectionCount atomic.Int64
|
||||
privateKey wgtypes.Key
|
||||
packetChan chan Packet
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
// Session tracking for WireGuard peers
|
||||
// Key format: "senderIndex:receiverIndex"
|
||||
@@ -406,6 +442,9 @@ func (s *UDPProxyServer) packetWorker() {
|
||||
logger.Debug("Created endpoint from packet remoteAddr %s: IP=%s, Port=%d", packet.remoteAddr.String(), endpoint.IP, endpoint.Port)
|
||||
|
||||
// Check if anything meaningful changed before queuing an HTTP notification.
|
||||
// The cache expires after 2.5 s so the server always receives a fresh
|
||||
// timestamp within its 5-second staleness window.
|
||||
const endpointCacheTTL = 2500 * time.Millisecond
|
||||
cacheKey := endpoint.OlmID + ":" + endpoint.NewtID
|
||||
newState := cachedEndpointState{
|
||||
OlmID: endpoint.OlmID,
|
||||
@@ -415,16 +454,19 @@ func (s *UDPProxyServer) packetWorker() {
|
||||
Port: endpoint.Port,
|
||||
PublicKey: endpoint.ClientPublicKey,
|
||||
}
|
||||
if cached, ok := s.lastEndpointCache.Load(cacheKey); ok && cached.(cachedEndpointState) == newState {
|
||||
// Endpoint unchanged - skip the HTTP call but still clear stale sessions.
|
||||
logger.Debug("Endpoint unchanged for %s, skipping notification", cacheKey)
|
||||
metrics.RecordHolePunchEvent(relayIfname, "deduplicated")
|
||||
s.clearSessionsForIP(endpoint.IP)
|
||||
metrics.RecordHolePunchEvent(relayIfname, "success")
|
||||
bufferPool.Put(packet.data[:1500])
|
||||
continue
|
||||
if cached, ok := s.lastEndpointCache.Load(cacheKey); ok {
|
||||
entry := cached.(cachedEndpointEntry)
|
||||
if entry.state == newState && time.Since(entry.cachedAt) < endpointCacheTTL {
|
||||
// Endpoint unchanged and cache still fresh - skip the HTTP call.
|
||||
logger.Debug("Endpoint unchanged for %s, skipping notification", cacheKey)
|
||||
metrics.RecordHolePunchEvent(relayIfname, "deduplicated")
|
||||
s.clearSessionsForIP(endpoint.IP)
|
||||
metrics.RecordHolePunchEvent(relayIfname, "success")
|
||||
bufferPool.Put(packet.data[:1500])
|
||||
continue
|
||||
}
|
||||
}
|
||||
s.lastEndpointCache.Store(cacheKey, newState)
|
||||
s.lastEndpointCache.Store(cacheKey, cachedEndpointEntry{state: newState, cachedAt: time.Now()})
|
||||
|
||||
// Queue the notification asynchronously so the hot path is not blocked by HTTP.
|
||||
select {
|
||||
@@ -849,10 +891,17 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
|
||||
// Check if we have an existing connection
|
||||
if conn, ok := s.connections.Load(key); ok {
|
||||
destConn := conn.(*DestinationConn)
|
||||
destConn.lastUsed = time.Now()
|
||||
destConn.lastUsed.Store(time.Now().UnixNano())
|
||||
return destConn.conn, nil
|
||||
}
|
||||
|
||||
// Enforce a hard cap on concurrent sockets so a burst of peer churn can't
|
||||
// exhaust the host's ephemeral ports/fds. Evict the least-recently-used
|
||||
// connection to make room instead of growing unbounded.
|
||||
if s.connectionCount.Load() >= maxUDPConnections {
|
||||
s.evictLRUConnection()
|
||||
}
|
||||
|
||||
// Create new connection
|
||||
newConn, err := net.DialUDP("udp", nil, destAddr)
|
||||
if err != nil {
|
||||
@@ -860,11 +909,17 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
|
||||
return nil, fmt.Errorf("failed to create UDP connection: %v", err)
|
||||
}
|
||||
|
||||
// Store the new connection
|
||||
s.connections.Store(key, &DestinationConn{
|
||||
conn: newConn,
|
||||
lastUsed: time.Now(),
|
||||
})
|
||||
destConn := &DestinationConn{conn: newConn}
|
||||
destConn.lastUsed.Store(time.Now().UnixNano())
|
||||
|
||||
// Store the new connection. If another goroutine raced us and already
|
||||
// created one for this key, close ours and use theirs instead.
|
||||
if existing, loaded := s.connections.LoadOrStore(key, destConn); loaded {
|
||||
newConn.Close()
|
||||
return existing.(*DestinationConn).conn, nil
|
||||
}
|
||||
s.connectionCount.Add(1)
|
||||
metrics.RecordUDPConnection(relayIfname, 1)
|
||||
|
||||
// Start a goroutine to handle responses
|
||||
go s.handleResponses(newConn, destAddr, remoteAddr)
|
||||
@@ -872,6 +927,33 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
|
||||
return newConn, nil
|
||||
}
|
||||
|
||||
// evictLRUConnection closes and removes the least-recently-used destination
|
||||
// connection so a new one can be created under the concurrent connection cap.
|
||||
func (s *UDPProxyServer) evictLRUConnection() {
|
||||
var oldestKey interface{}
|
||||
var oldestConn *DestinationConn
|
||||
var oldestTime int64
|
||||
|
||||
s.connections.Range(func(key, value interface{}) bool {
|
||||
destConn := value.(*DestinationConn)
|
||||
lu := destConn.lastUsed.Load()
|
||||
if oldestKey == nil || lu < oldestTime {
|
||||
oldestKey = key
|
||||
oldestConn = destConn
|
||||
oldestTime = lu
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if oldestKey != nil {
|
||||
s.connections.Delete(oldestKey)
|
||||
oldestConn.conn.Close()
|
||||
s.connectionCount.Add(-1)
|
||||
metrics.RecordUDPConnection(relayIfname, -1)
|
||||
metrics.RecordProxyCleanupRemoved(relayIfname, "conn_evicted", 1)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *UDPProxyServer) handleResponses(conn *net.UDPConn, destAddr *net.UDPAddr, remoteAddr *net.UDPAddr) {
|
||||
buffer := make([]byte, 1500)
|
||||
for {
|
||||
@@ -923,22 +1005,32 @@ func (s *UDPProxyServer) handleResponses(conn *net.UDPConn, destAddr *net.UDPAdd
|
||||
|
||||
// Add a cleanup method to periodically remove idle connections
|
||||
func (s *UDPProxyServer) cleanupIdleConnections() {
|
||||
ticker := time.NewTicker(5 * time.Minute)
|
||||
// Ticker interval and idle threshold were previously 5min/10min, meaning
|
||||
// a socket could sit open for up to 15 minutes after going idle. Under a
|
||||
// reconnect/churn burst that lag is enough to exhaust ephemeral ports
|
||||
// before cleanup catches up, so both are tightened here.
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
cleanupStart := time.Now()
|
||||
now := time.Now()
|
||||
now := time.Now().UnixNano()
|
||||
removed := int64(0)
|
||||
s.connections.Range(func(key, value interface{}) bool {
|
||||
destConn := value.(*DestinationConn)
|
||||
if now.Sub(destConn.lastUsed) > 10*time.Minute {
|
||||
if now-destConn.lastUsed.Load() > int64(5*time.Minute) {
|
||||
destConn.conn.Close()
|
||||
s.connections.Delete(key)
|
||||
metrics.RecordProxyCleanupRemoved(relayIfname, "conn", 1)
|
||||
removed++
|
||||
}
|
||||
return true
|
||||
})
|
||||
if removed > 0 {
|
||||
s.connectionCount.Add(-removed)
|
||||
metrics.RecordUDPConnection(relayIfname, -removed)
|
||||
metrics.RecordProxyCleanupRemoved(relayIfname, "conn", removed)
|
||||
}
|
||||
metrics.RecordProxyIdleCleanupDuration(relayIfname, "conn", time.Since(cleanupStart).Seconds())
|
||||
case <-s.ctx.Done():
|
||||
return
|
||||
@@ -1100,6 +1192,10 @@ func (s *UDPProxyServer) clearConnectionsForWGIP(wgIP string) {
|
||||
for _, key := range keysToDelete {
|
||||
s.connections.Delete(key)
|
||||
}
|
||||
if len(keysToDelete) > 0 {
|
||||
s.connectionCount.Add(-int64(len(keysToDelete)))
|
||||
metrics.RecordUDPConnection(relayIfname, -int64(len(keysToDelete)))
|
||||
}
|
||||
|
||||
logger.Info("Cleared %d connections for WG IP: %s", len(keysToDelete), wgIP)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user