5 Commits

Author SHA1 Message Date
Laurence
390b401998 Merge remote-tracking branch 'upstream/dev' into proxy-context-tunnel-tracking 2026-04-09 15:03:00 +01:00
Laurence
5eacbb7239 fix(proxy): prevent deleting wrong tunnel in defer cleanup
Add pointer check before delete to handle race where UpdateLocalSNIs
removes our tunnel and a new one is created for the same hostname.
2026-03-13 16:43:16 +00:00
Laurence
d21c09c84f refactor(proxy): simplify tunnel tracking with mutex-only approach
Remove atomic counter in favor of simple int protected by mutex.
Eliminates race condition complexity and recheck logic.
2026-03-13 16:36:56 +00:00
Laurence
28c65b950c fix(proxy): avoid shadowing ctx variable in pipe() 2026-03-13 15:51:23 +00:00
Laurence
1643d71905 refactor(proxy): use context cancellation for tunnel tracking
- Replace []net.Conn slice with context + atomic counter in activeTunnel
- Use errgroup.WithContext for pipe() to handle goroutine lifecycle
- Use context.AfterFunc to close connections on cancellation
- Fix race condition by comparing tunnel pointers instead of map lookup
- UpdateLocalSNIs now cancels tunnel context instead of iterating conns

This eliminates O(n) connection removal, prevents goroutine leaks,
and provides cleaner cancellation semantics.
2026-03-13 15:47:52 +00:00
29 changed files with 472 additions and 1972 deletions

View File

@@ -14,13 +14,12 @@ body:
label: Environment
description: Please fill out the relevant details below for your environment.
value: |
- OS Type & Version:
- OS Type & Version: (e.g., Ubuntu 22.04)
- Pangolin Version:
- Edition (Community or Enterprise):
- Gerbil Version:
- Traefik Version:
- Newt Version:
- Client Version:
- Olm Version: (if applicable)
validations:
required: true

View File

@@ -1,32 +1,40 @@
version: 2
updates:
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "daily"
open-pull-requests-limit: 1
groups:
go-dependencies:
patterns:
- "*"
dev-patch-updates:
dependency-type: "development"
update-types:
- "patch"
dev-minor-updates:
dependency-type: "development"
update-types:
- "minor"
prod-patch-updates:
dependency-type: "production"
update-types:
- "patch"
prod-minor-updates:
dependency-type: "production"
update-types:
- "minor"
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "daily"
open-pull-requests-limit: 1
groups:
docker-dependencies:
patterns:
- "*"
patch-updates:
update-types:
- "patch"
minor-updates:
update-types:
- "minor"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
open-pull-requests-limit: 1
groups:
github-actions-dependencies:
patterns:
- "*"

View File

@@ -36,16 +36,16 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
- name: Log in to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
with:
registry: docker.io
username: ${{ secrets.DOCKER_HUB_USERNAME }}
@@ -57,9 +57,9 @@ jobs:
shell: bash
- name: Install Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
with:
go-version: 1.26
go-version: 1.25
- name: Update version in main.go
run: |
@@ -80,7 +80,7 @@ jobs:
shell: bash
- name: Login in to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -107,9 +107,8 @@ jobs:
shell: bash
- name: Install cosign
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
with:
cosign-release: v3.0.6
# cosign is used to sign and verify container images (key and keyless)
uses: sigstore/cosign-installer@ba7bc0a3fef59531c69a25acd34668d6d3fe6f22 # v4.1.0
- name: Dual-sign and verify (GHCR & Docker Hub)
# Sign each image by digest using keyless (OIDC) and key-based signing,
@@ -156,7 +155,7 @@ jobs:
shell: bash
- name: Upload artifacts from /bin
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: binaries
path: bin/

View File

@@ -23,7 +23,7 @@ jobs:
skopeo --version
- name: Install cosign
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
uses: sigstore/cosign-installer@ba7bc0a3fef59531c69a25acd34668d6d3fe6f22 # v4.1.0
- name: Input check
run: |

View File

@@ -14,16 +14,12 @@ jobs:
runs-on: amd64-runner
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Read go version
id: goversion
run: echo "version=$(cat .go-version)" >> $GITHUB_OUTPUT
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
with:
go-version: ${{ steps.goversion.outputs.version }}
go-version: 1.26
- name: Build go
run: go build

View File

@@ -1 +1 @@
1.26
1.25

View File

@@ -1,5 +1,4 @@
ARG GO_VERSION=1.26
FROM golang:${GO_VERSION}-alpine AS builder
FROM golang:1.26-alpine AS builder
# Set the working directory inside the container
WORKDIR /app
@@ -17,7 +16,7 @@ COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o /gerbil
# Start a new stage from scratch
FROM alpine:3.24 AS runner
FROM alpine:3.23 AS runner
RUN apk add --no-cache iptables iproute2
@@ -26,4 +25,4 @@ COPY entrypoint.sh /
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
CMD ["gerbil"]
CMD ["gerbil"]

View File

@@ -1,6 +1,4 @@
GO_VERSION := $(shell cat .go-version 2>/dev/null)
all: build push
docker-build-release:
@@ -9,10 +7,10 @@ docker-build-release:
exit 1; \
fi
docker buildx build --platform linux/arm64,linux/amd64 -t fosrl/gerbil:latest -f Dockerfile --push .
docker buildx build --platform linux/arm64,linux/amd64 -t fosrl/gerbil:$(tag) -f Dockerfile --build-arg GO_VERSION=$(GO_VERSION) --push .
docker buildx build --platform linux/arm64,linux/amd64 -t fosrl/gerbil:$(tag) -f Dockerfile --push .
build:
docker build -t fosrl/gerbil:latest --build-arg GO_VERSION=$(GO_VERSION) .
docker build -t fosrl/gerbil:latest .
push:
docker push fosrl/gerbil:latest

View File

@@ -138,7 +138,7 @@ make
### Binary
Make sure to have Go 1.26 installed.
Make sure to have Go 1.23.1 installed.
```bash
make local

View File

@@ -83,7 +83,6 @@ type OTelConfig struct {
Endpoint string // default: "localhost:4317"
Insecure bool // default: true
ExportInterval time.Duration // default: 60s
Timeout time.Duration // default: 10s
}
```
@@ -98,7 +97,6 @@ type OTelConfig struct {
| `OTEL_METRICS_ENDPOINT` | `localhost:4317` | OTLP collector address |
| `OTEL_METRICS_INSECURE` | `true` | Disable TLS for OTLP |
| `OTEL_METRICS_EXPORT_INTERVAL` | `60s` | Push interval (e.g. `10s`, `1m`) |
| `OTEL_METRICS_TIMEOUT` | `10s` | Timeout for OTLP exporter connection setup |
| `DEPLOYMENT_ENVIRONMENT` | _(unset)_ | OTel deployment.environment attribute |
### CLI flags
@@ -110,8 +108,7 @@ type OTelConfig struct {
--otel-metrics-protocol string (default: grpc)
--otel-metrics-endpoint string (default: localhost:4317)
--otel-metrics-insecure bool (default: true)
--otel-metrics-export-interval duration (default: 60s)
--otel-metrics-timeout duration (default: 10s)
--otel-metrics-export-interval duration (default: 1m0s)
```
---
@@ -167,7 +164,6 @@ export OTEL_METRICS_PROTOCOL=grpc
export OTEL_METRICS_ENDPOINT=otel-collector:4317
export OTEL_METRICS_INSECURE=true
export OTEL_METRICS_EXPORT_INTERVAL=10s
export OTEL_METRICS_TIMEOUT=10s
export DEPLOYMENT_ENVIRONMENT=production
```
@@ -180,7 +176,6 @@ export DEPLOYMENT_ENVIRONMENT=production
--otel-metrics-endpoint=otel-collector:4317 \
--otel-metrics-insecure \
--otel-metrics-export-interval=10s \
--otel-metrics-timeout=10s \
--config=/etc/gerbil/config.json
```
@@ -230,6 +225,7 @@ All metrics use the prefix `gerbil_<component>_<name>`.
| Metric | Type | Labels |
|--------|------|--------|
| `gerbil_proxy_mapping_active` | UpDownCounter | `ifname` |
| `gerbil_session_active` | UpDownCounter | `ifname` |
| `gerbil_active_sessions` | UpDownCounter | `ifname` |
| `gerbil_udp_packets_total` | Counter | `ifname`, `type`, `direction` |
| `gerbil_hole_punch_events_total` | Counter | `ifname`, `result` |
@@ -260,7 +256,7 @@ The `docker-compose.metrics.yml` provides a complete observability stack.
**Prometheus mode:**
```bash
METRICS_BACKEND=prometheus docker-compose -f docker compose.metrics.yml up -d
METRICS_BACKEND=prometheus docker-compose -f docker-compose.metrics.yml up -d
# Scrape at http://localhost:3003/metrics
# Grafana at http://localhost:3000 (admin/admin)
```
@@ -269,5 +265,5 @@ METRICS_BACKEND=prometheus docker-compose -f docker compose.metrics.yml up -d
```bash
METRICS_BACKEND=otel OTEL_METRICS_ENDPOINT=otel-collector:4317 \
docker compose -f docker-compose.metrics.yml up -d
docker-compose -f docker-compose.metrics.yml up -d
```

View File

@@ -1,4 +1,3 @@
file_format: '1.0'
receivers:
otlp:
protocols:
@@ -44,4 +43,4 @@ service:
metrics:
receivers: [otlp]
processors: [batch, resource]
exporters: [prometheus, prometheusremotewrite, debug]
exporters: [prometheus, prometheusremotewrite, debug]

47
go.mod
View File

@@ -4,16 +4,16 @@ go 1.26.0
require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/prometheus/client_golang v1.24.1
github.com/prometheus/client_golang v1.20.5
github.com/vishvananda/netlink v1.3.1
go.opentelemetry.io/otel v1.46.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0
go.opentelemetry.io/otel/metric v1.46.0
go.opentelemetry.io/otel/sdk v1.46.0
go.opentelemetry.io/otel/sdk/metric v1.46.0
golang.org/x/crypto v0.55.0
golang.org/x/sync v0.22.0
go.opentelemetry.io/otel v1.42.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0
go.opentelemetry.io/otel/metric v1.42.0
go.opentelemetry.io/otel/sdk v1.42.0
go.opentelemetry.io/otel/sdk/metric v1.42.0
golang.org/x/crypto v0.49.0
golang.org/x/sync v0.20.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6
)
@@ -21,29 +21,30 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/josharian/native v1.1.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/mdlayher/genetlink v1.3.2 // indirect
github.com/mdlayher/netlink v1.7.2 // indirect
github.com/mdlayher/socket v0.4.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.61.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/trace v1.46.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
go.opentelemetry.io/otel/trace v1.42.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
golang.org/x/net v0.51.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/text v0.35.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
google.golang.org/grpc v1.83.1 // indirect
google.golang.org/protobuf v1.36.12 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect
google.golang.org/grpc v1.79.3 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)

118
go.sum
View File

@@ -4,9 +4,11 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
@@ -15,12 +17,12 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
@@ -35,69 +37,65 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ=
github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 h1:AP23h/mFgb/lc7tdck1Kfn9qxsM8TAeNPCU5C3pzaps=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0/go.mod h1:K4EqCe1b4kGk5WR690ntg9LaBfsPoV32FwthbyoptuA=
go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU=
go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4=
go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0 h1:H7O6RlGOMTizyl3R08Kn5pdM06bnH8oscSj7o11tmLA=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0/go.mod h1:mBFWu/WOVDkWWsR7Tx7h6EpQB8wsv7P0Yrh0Pb7othc=
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 h1:CawjfCvYQH2OU3/TnxLx97WDSUDRABfT18pCOYwc2GE=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6/go.mod h1:3rxYc4HtVcSG9gVaTs2GEBdehh+sYPOwKtyUWEOTb80=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI=
google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0=
google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -7,9 +7,7 @@ package metrics
import (
"context"
"fmt"
"net/http"
"sync"
"github.com/fosrl/gerbil/internal/observability"
)
@@ -26,7 +24,6 @@ type OTelConfig = observability.OTelConfig
var (
backend observability.Backend
initMu sync.Mutex
// Interface and peer metrics
wgInterfaceUp observability.Int64Gauge
@@ -58,7 +55,7 @@ var (
udpPacketSizeBytes observability.Histogram
holePunchEventsTotal observability.Counter
proxyMappingActive observability.UpDownCounter
relayUDPConnectionsActive observability.UpDownCounter
sessionActive observability.UpDownCounter
sessionRebuiltTotal observability.Counter
commPatternActive observability.UpDownCounter
proxyCleanupRemovedTotal observability.Counter
@@ -110,13 +107,6 @@ func DefaultConfig() Config {
// Initialize sets up the metrics system using the selected backend.
// It returns the /metrics HTTP handler (non-nil only for Prometheus backend).
func Initialize(cfg Config) (http.Handler, error) {
initMu.Lock()
defer initMu.Unlock()
if backend != nil {
return backend.HTTPHandler(), nil
}
b, err := observability.New(cfg)
if err != nil {
return nil, err
@@ -124,7 +114,6 @@ func Initialize(cfg Config) (http.Handler, error) {
backend = b
if err := createInstruments(); err != nil {
backend = nil
return nil, err
}
@@ -133,13 +122,8 @@ func Initialize(cfg Config) (http.Handler, error) {
// Shutdown gracefully shuts down the metrics backend.
func Shutdown(ctx context.Context) error {
initMu.Lock()
b := backend
backend = nil
initMu.Unlock()
if b != nil {
return b.Shutdown(ctx)
if backend != nil {
return backend.Shutdown(ctx)
}
return nil
}
@@ -151,351 +135,129 @@ func createInstruments() error {
b := backend
newCounter := func(name, desc string, labelNames ...string) (observability.Counter, error) {
c, err := b.NewCounter(name, desc, labelNames...)
if err != nil {
return nil, fmt.Errorf("create counter %q: %w", name, err)
}
return c, nil
}
newUpDownCounter := func(name, desc string, labelNames ...string) (observability.UpDownCounter, error) {
c, err := b.NewUpDownCounter(name, desc, labelNames...)
if err != nil {
return nil, fmt.Errorf("create updown counter %q: %w", name, err)
}
return c, nil
}
newInt64Gauge := func(name, desc string, labelNames ...string) (observability.Int64Gauge, error) {
g, err := b.NewInt64Gauge(name, desc, labelNames...)
if err != nil {
return nil, fmt.Errorf("create int64 gauge %q: %w", name, err)
}
return g, nil
}
newFloat64Gauge := func(name, desc string, labelNames ...string) (observability.Float64Gauge, error) {
g, err := b.NewFloat64Gauge(name, desc, labelNames...)
if err != nil {
return nil, fmt.Errorf("create float64 gauge %q: %w", name, err)
}
return g, nil
}
newHistogram := func(name, desc string, buckets []float64, labelNames ...string) (observability.Histogram, error) {
h, err := b.NewHistogram(name, desc, buckets, labelNames...)
if err != nil {
return nil, fmt.Errorf("create histogram %q: %w", name, err)
}
return h, nil
}
var err error
wgInterfaceUp, err = newInt64Gauge("gerbil_wg_interface_up",
wgInterfaceUp = b.NewInt64Gauge("gerbil_wg_interface_up",
"Operational state of a WireGuard interface (1=up, 0=down)", "ifname", "instance")
if err != nil {
return err
}
wgPeersTotal, err = newUpDownCounter("gerbil_wg_peers_total",
wgPeersTotal = b.NewUpDownCounter("gerbil_wg_peers_total",
"Total number of configured peers per interface", "ifname")
if err != nil {
return err
}
wgPeerConnected, err = newInt64Gauge("gerbil_wg_peer_connected",
wgPeerConnected = b.NewInt64Gauge("gerbil_wg_peer_connected",
"Whether a specific peer is connected (1=connected, 0=disconnected)", "ifname", "peer")
if err != nil {
return err
}
allowedIPsCount, err = newUpDownCounter("gerbil_allowed_ips_count",
allowedIPsCount = b.NewUpDownCounter("gerbil_allowed_ips_count",
"Number of allowed IPs configured per peer", "ifname", "peer")
if err != nil {
return err
}
keyRotationTotal, err = newCounter("gerbil_key_rotation_total",
keyRotationTotal = b.NewCounter("gerbil_key_rotation_total",
"Key rotation events", "ifname", "reason")
if err != nil {
return err
}
wgHandshakesTotal, err = newCounter("gerbil_wg_handshakes_total",
wgHandshakesTotal = b.NewCounter("gerbil_wg_handshakes_total",
"Count of handshake attempts with their result status", "ifname", "peer", "result")
if err != nil {
return err
}
wgHandshakeLatency, err = newHistogram("gerbil_wg_handshake_latency_seconds",
wgHandshakeLatency = b.NewHistogram("gerbil_wg_handshake_latency_seconds",
"Distribution of handshake latencies in seconds", durationBuckets, "ifname", "peer")
if err != nil {
return err
}
wgPeerRTT, err = newHistogram("gerbil_wg_peer_rtt_seconds",
wgPeerRTT = b.NewHistogram("gerbil_wg_peer_rtt_seconds",
"Observed round-trip time to a peer in seconds", durationBuckets, "ifname", "peer")
if err != nil {
return err
}
wgBytesReceived, err = newCounter("gerbil_wg_bytes_received_total",
wgBytesReceived = b.NewCounter("gerbil_wg_bytes_received_total",
"Number of bytes received from a peer", "ifname", "peer")
if err != nil {
return err
}
wgBytesTransmitted, err = newCounter("gerbil_wg_bytes_transmitted_total",
wgBytesTransmitted = b.NewCounter("gerbil_wg_bytes_transmitted_total",
"Number of bytes transmitted to a peer", "ifname", "peer")
if err != nil {
return err
}
netlinkEventsTotal, err = newCounter("gerbil_netlink_events_total",
netlinkEventsTotal = b.NewCounter("gerbil_netlink_events_total",
"Number of netlink events processed", "event_type")
if err != nil {
return err
}
netlinkErrorsTotal, err = newCounter("gerbil_netlink_errors_total",
netlinkErrorsTotal = b.NewCounter("gerbil_netlink_errors_total",
"Count of netlink or kernel errors", "component", "error_type")
if err != nil {
return err
}
syncDuration, err = newHistogram("gerbil_sync_duration_seconds",
syncDuration = b.NewHistogram("gerbil_sync_duration_seconds",
"Duration of reconciliation/sync loops in seconds", durationBuckets, "component")
if err != nil {
return err
}
workqueueDepth, err = newUpDownCounter("gerbil_workqueue_depth",
workqueueDepth = b.NewUpDownCounter("gerbil_workqueue_depth",
"Current length of internal work queues", "queue")
if err != nil {
return err
}
kernelModuleLoads, err = newCounter("gerbil_kernel_module_loads_total",
kernelModuleLoads = b.NewCounter("gerbil_kernel_module_loads_total",
"Count of kernel module load attempts", "result")
if err != nil {
return err
}
firewallRulesApplied, err = newCounter("gerbil_firewall_rules_applied_total",
firewallRulesApplied = b.NewCounter("gerbil_firewall_rules_applied_total",
"IPTables/NFT rules applied", "result", "chain")
if err != nil {
return err
}
activeSessions, err = newUpDownCounter("gerbil_active_sessions",
activeSessions = b.NewUpDownCounter("gerbil_active_sessions",
"Number of active UDP relay sessions", "ifname")
if err != nil {
return err
}
activeProxyConnections, err = newUpDownCounter("gerbil_active_proxy_connections",
activeProxyConnections = b.NewUpDownCounter("gerbil_active_proxy_connections",
"Active SNI proxy connections")
if err != nil {
return err
}
proxyRouteLookups, err = newCounter("gerbil_proxy_route_lookups_total",
proxyRouteLookups = b.NewCounter("gerbil_proxy_route_lookups_total",
"Number of route lookups", "result")
if err != nil {
return err
}
proxyTLSHandshake, err = newHistogram("gerbil_proxy_tls_handshake_seconds",
proxyTLSHandshake = b.NewHistogram("gerbil_proxy_tls_handshake_seconds",
"TLS handshake duration for SNI proxy in seconds", durationBuckets)
if err != nil {
return err
}
proxyBytesTransmitted, err = newCounter("gerbil_proxy_bytes_transmitted_total",
proxyBytesTransmitted = b.NewCounter("gerbil_proxy_bytes_transmitted_total",
"Bytes sent/received by the SNI proxy", "direction")
if err != nil {
return err
}
configReloadsTotal, err = newCounter("gerbil_config_reloads_total",
configReloadsTotal = b.NewCounter("gerbil_config_reloads_total",
"Number of configuration reloads", "result")
if err != nil {
return err
}
restartTotal, err = newCounter("gerbil_restart_total",
restartTotal = b.NewCounter("gerbil_restart_total",
"Process restart count")
if err != nil {
return err
}
authFailuresTotal, err = newCounter("gerbil_auth_failures_total",
authFailuresTotal = b.NewCounter("gerbil_auth_failures_total",
"Count of authentication or peer validation failures", "peer", "reason")
if err != nil {
return err
}
aclDeniedTotal, err = newCounter("gerbil_acl_denied_total",
aclDeniedTotal = b.NewCounter("gerbil_acl_denied_total",
"Access control denied events", "ifname", "peer", "policy")
if err != nil {
return err
}
certificateExpiryDays, err = newFloat64Gauge("gerbil_certificate_expiry_days",
certificateExpiryDays = b.NewFloat64Gauge("gerbil_certificate_expiry_days",
"Days until certificate expiry", "cert_name", "ifname")
if err != nil {
return err
}
udpPacketsTotal, err = newCounter("gerbil_udp_packets_total",
udpPacketsTotal = b.NewCounter("gerbil_udp_packets_total",
"Count of UDP packets processed by relay workers", "ifname", "type", "direction")
if err != nil {
return err
}
udpPacketSizeBytes, err = newHistogram("gerbil_udp_packet_size_bytes",
udpPacketSizeBytes = b.NewHistogram("gerbil_udp_packet_size_bytes",
"Size distribution of packets forwarded through relay", sizeBuckets, "ifname", "type")
if err != nil {
return err
}
holePunchEventsTotal, err = newCounter("gerbil_hole_punch_events_total",
holePunchEventsTotal = b.NewCounter("gerbil_hole_punch_events_total",
"Count of hole punch messages processed", "ifname", "result")
if err != nil {
return err
}
proxyMappingActive, err = newUpDownCounter("gerbil_proxy_mapping_active",
proxyMappingActive = b.NewUpDownCounter("gerbil_proxy_mapping_active",
"Number of active proxy mappings", "ifname")
if err != nil {
return err
}
relayUDPConnectionsActive, err = newUpDownCounter("gerbil_relay_udp_connections_active",
"Number of open per-peer outbound UDP sockets held by the relay connection pool", "ifname")
if err != nil {
return err
}
sessionRebuiltTotal, err = newCounter("gerbil_session_rebuilt_total",
sessionActive = b.NewUpDownCounter("gerbil_session_active",
"Number of active WireGuard sessions", "ifname")
sessionRebuiltTotal = b.NewCounter("gerbil_session_rebuilt_total",
"Count of sessions rebuilt from communication patterns", "ifname")
if err != nil {
return err
}
commPatternActive, err = newUpDownCounter("gerbil_comm_pattern_active",
commPatternActive = b.NewUpDownCounter("gerbil_comm_pattern_active",
"Number of active communication patterns", "ifname")
if err != nil {
return err
}
proxyCleanupRemovedTotal, err = newCounter("gerbil_proxy_cleanup_removed_total",
proxyCleanupRemovedTotal = b.NewCounter("gerbil_proxy_cleanup_removed_total",
"Count of items removed during cleanup routines", "ifname", "component")
if err != nil {
return err
}
proxyConnectionErrorsTotal, err = newCounter("gerbil_proxy_connection_errors_total",
proxyConnectionErrorsTotal = b.NewCounter("gerbil_proxy_connection_errors_total",
"Count of connection errors in proxy operations", "ifname", "error_type")
if err != nil {
return err
}
proxyInitialMappingsTotal, err = newInt64Gauge("gerbil_proxy_initial_mappings",
proxyInitialMappingsTotal = b.NewInt64Gauge("gerbil_proxy_initial_mappings",
"Number of initial proxy mappings loaded", "ifname")
if err != nil {
return err
}
proxyMappingUpdatesTotal, err = newCounter("gerbil_proxy_mapping_updates_total",
proxyMappingUpdatesTotal = b.NewCounter("gerbil_proxy_mapping_updates_total",
"Count of proxy mapping updates", "ifname")
if err != nil {
return err
}
proxyIdleCleanupDuration, err = newHistogram("gerbil_proxy_idle_cleanup_duration_seconds",
proxyIdleCleanupDuration = b.NewHistogram("gerbil_proxy_idle_cleanup_duration_seconds",
"Duration of cleanup cycles", durationBuckets, "ifname", "component")
if err != nil {
return err
}
sniConnectionsTotal, err = newCounter("gerbil_sni_connections_total",
sniConnectionsTotal = b.NewCounter("gerbil_sni_connections_total",
"Count of connections processed by SNI proxy", "result")
if err != nil {
return err
}
sniConnectionDuration, err = newHistogram("gerbil_sni_connection_duration_seconds",
sniConnectionDuration = b.NewHistogram("gerbil_sni_connection_duration_seconds",
"Lifetime distribution of proxied TLS connections", sniDurationBuckets)
if err != nil {
return err
}
sniActiveConnections, err = newUpDownCounter("gerbil_sni_active_connections",
sniActiveConnections = b.NewUpDownCounter("gerbil_sni_active_connections",
"Number of active SNI tunnels")
if err != nil {
return err
}
sniRouteCacheHitsTotal, err = newCounter("gerbil_sni_route_cache_hits_total",
sniRouteCacheHitsTotal = b.NewCounter("gerbil_sni_route_cache_hits_total",
"Count of route cache hits and misses", "result")
if err != nil {
return err
}
sniRouteAPIRequestsTotal, err = newCounter("gerbil_sni_route_api_requests_total",
sniRouteAPIRequestsTotal = b.NewCounter("gerbil_sni_route_api_requests_total",
"Count of route API requests", "result")
if err != nil {
return err
}
sniRouteAPILatency, err = newHistogram("gerbil_sni_route_api_latency_seconds",
sniRouteAPILatency = b.NewHistogram("gerbil_sni_route_api_latency_seconds",
"Distribution of route API call latencies", durationBuckets)
if err != nil {
return err
}
sniLocalOverrideTotal, err = newCounter("gerbil_sni_local_override_total",
sniLocalOverrideTotal = b.NewCounter("gerbil_sni_local_override_total",
"Count of routes using local overrides", "hit")
if err != nil {
return err
}
sniTrustedProxyEventsTotal, err = newCounter("gerbil_sni_trusted_proxy_events_total",
sniTrustedProxyEventsTotal = b.NewCounter("gerbil_sni_trusted_proxy_events_total",
"Count of PROXY protocol events", "event")
if err != nil {
return err
}
sniProxyProtocolParseErrorsTotal, err = newCounter("gerbil_sni_proxy_protocol_parse_errors_total",
sniProxyProtocolParseErrorsTotal = b.NewCounter("gerbil_sni_proxy_protocol_parse_errors_total",
"Count of PROXY protocol parse failures")
if err != nil {
return err
}
sniDataBytesTotal, err = newCounter("gerbil_sni_data_bytes_total",
sniDataBytesTotal = b.NewCounter("gerbil_sni_data_bytes_total",
"Count of bytes proxied through SNI tunnels", "direction")
if err != nil {
return err
}
sniTunnelTerminationsTotal, err = newCounter("gerbil_sni_tunnel_terminations_total",
sniTunnelTerminationsTotal = b.NewCounter("gerbil_sni_tunnel_terminations_total",
"Count of tunnel terminations by reason", "reason")
if err != nil {
return err
}
httpRequestsTotal, err = newCounter("gerbil_http_requests_total",
httpRequestsTotal = b.NewCounter("gerbil_http_requests_total",
"Count of HTTP requests to management API", "endpoint", "method", "status_code")
if err != nil {
return err
}
httpRequestDuration, err = newHistogram("gerbil_http_request_duration_seconds",
httpRequestDuration = b.NewHistogram("gerbil_http_request_duration_seconds",
"Distribution of HTTP request handling time", durationBuckets, "endpoint", "method")
if err != nil {
return err
}
peerOperationsTotal, err = newCounter("gerbil_peer_operations_total",
peerOperationsTotal = b.NewCounter("gerbil_peer_operations_total",
"Count of peer lifecycle operations", "operation", "result")
if err != nil {
return err
}
proxyMappingUpdateRequestsTotal, err = newCounter("gerbil_proxy_mapping_update_requests_total",
proxyMappingUpdateRequestsTotal = b.NewCounter("gerbil_proxy_mapping_update_requests_total",
"Count of proxy mapping update API calls", "result")
if err != nil {
return err
}
destinationsUpdateRequestsTotal, err = newCounter("gerbil_destinations_update_requests_total",
destinationsUpdateRequestsTotal = b.NewCounter("gerbil_destinations_update_requests_total",
"Count of destinations update API calls", "result")
if err != nil {
return err
}
remoteConfigFetchesTotal, err = newCounter("gerbil_remote_config_fetches_total",
remoteConfigFetchesTotal = b.NewCounter("gerbil_remote_config_fetches_total",
"Count of remote configuration fetch attempts", "result")
if err != nil {
return err
}
bandwidthReportsTotal, err = newCounter("gerbil_bandwidth_reports_total",
bandwidthReportsTotal = b.NewCounter("gerbil_bandwidth_reports_total",
"Count of bandwidth report transmissions", "result")
if err != nil {
return err
}
peerBandwidthBytesTotal, err = newCounter("gerbil_peer_bandwidth_bytes_total",
peerBandwidthBytesTotal = b.NewCounter("gerbil_peer_bandwidth_bytes_total",
"Bytes per peer tracked by bandwidth calculation", "peer", "direction")
if err != nil {
return err
}
memorySpikeTotal, err = newCounter("gerbil_memory_spike_total",
memorySpikeTotal = b.NewCounter("gerbil_memory_spike_total",
"Count of memory spikes detected", "severity")
if err != nil {
return err
}
heapProfilesWrittenTotal, err = newCounter("gerbil_heap_profiles_written_total",
heapProfilesWrittenTotal = b.NewCounter("gerbil_heap_profiles_written_total",
"Count of heap profile files generated")
if err != nil {
return err
}
return nil
}
func RecordInterfaceUp(ifname, instance string, up bool) {
if wgInterfaceUp == nil {
return
}
value := int64(0)
if up {
value = 1
@@ -504,16 +266,10 @@ func RecordInterfaceUp(ifname, instance string, up bool) {
}
func RecordPeersTotal(ifname string, delta int64) {
if wgPeersTotal == nil {
return
}
wgPeersTotal.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
}
func RecordPeerConnected(ifname, peer string, connected bool) {
if wgPeerConnected == nil {
return
}
value := int64(0)
if connected {
value = 1
@@ -522,400 +278,229 @@ func RecordPeerConnected(ifname, peer string, connected bool) {
}
func RecordHandshake(ifname, peer, result string) {
if wgHandshakesTotal == nil {
return
}
wgHandshakesTotal.Add(context.Background(), 1, observability.Labels{"ifname": ifname, "peer": peer, "result": result})
}
func RecordHandshakeLatency(ifname, peer string, seconds float64) {
if wgHandshakeLatency == nil {
return
}
wgHandshakeLatency.Record(context.Background(), seconds, observability.Labels{"ifname": ifname, "peer": peer})
}
func RecordPeerRTT(ifname, peer string, seconds float64) {
if wgPeerRTT == nil {
return
}
wgPeerRTT.Record(context.Background(), seconds, observability.Labels{"ifname": ifname, "peer": peer})
}
func RecordBytesReceived(ifname, peer string, bytes int64) {
if wgBytesReceived == nil {
return
}
wgBytesReceived.Add(context.Background(), bytes, observability.Labels{"ifname": ifname, "peer": peer})
}
func RecordBytesTransmitted(ifname, peer string, bytes int64) {
if wgBytesTransmitted == nil {
return
}
wgBytesTransmitted.Add(context.Background(), bytes, observability.Labels{"ifname": ifname, "peer": peer})
}
func RecordAllowedIPsCount(ifname, peer string, delta int64) {
if allowedIPsCount == nil {
return
}
allowedIPsCount.Add(context.Background(), delta, observability.Labels{"ifname": ifname, "peer": peer})
}
func RecordKeyRotation(ifname, reason string) {
if keyRotationTotal == nil {
return
}
keyRotationTotal.Add(context.Background(), 1, observability.Labels{"ifname": ifname, "reason": reason})
}
func RecordNetlinkEvent(eventType string) {
if netlinkEventsTotal == nil {
return
}
netlinkEventsTotal.Add(context.Background(), 1, observability.Labels{"event_type": eventType})
}
func RecordNetlinkError(component, errorType string) {
if netlinkErrorsTotal == nil {
return
}
netlinkErrorsTotal.Add(context.Background(), 1, observability.Labels{"component": component, "error_type": errorType})
}
func RecordSyncDuration(component string, seconds float64) {
if syncDuration == nil {
return
}
syncDuration.Record(context.Background(), seconds, observability.Labels{"component": component})
}
func RecordWorkqueueDepth(queue string, delta int64) {
if workqueueDepth == nil {
return
}
workqueueDepth.Add(context.Background(), delta, observability.Labels{"queue": queue})
}
func RecordKernelModuleLoad(result string) {
if kernelModuleLoads == nil {
return
}
kernelModuleLoads.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordFirewallRuleApplied(result, chain string) {
if firewallRulesApplied == nil {
return
}
firewallRulesApplied.Add(context.Background(), 1, observability.Labels{"result": result, "chain": chain})
}
func RecordActiveSession(ifname string, delta int64) {
if activeSessions == nil {
return
}
activeSessions.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
}
func RecordActiveProxyConnection(delta int64) {
if activeProxyConnections == nil {
return
}
func RecordActiveProxyConnection(hostname string, delta int64) {
_ = hostname
activeProxyConnections.Add(context.Background(), delta, nil)
}
func RecordProxyRouteLookup(result string) {
if proxyRouteLookups == nil {
return
}
func RecordProxyRouteLookup(result, hostname string) {
_ = hostname
proxyRouteLookups.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordProxyTLSHandshake(seconds float64) {
if proxyTLSHandshake == nil {
return
}
func RecordProxyTLSHandshake(hostname string, seconds float64) {
_ = hostname
proxyTLSHandshake.Record(context.Background(), seconds, nil)
}
func RecordProxyBytesTransmitted(direction string, bytes int64) {
if proxyBytesTransmitted == nil {
return
}
func RecordProxyBytesTransmitted(hostname, direction string, bytes int64) {
_ = hostname
proxyBytesTransmitted.Add(context.Background(), bytes, observability.Labels{"direction": direction})
}
func RecordConfigReload(result string) {
if configReloadsTotal == nil {
return
}
configReloadsTotal.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordRestart() {
if restartTotal == nil {
return
}
restartTotal.Add(context.Background(), 1, nil)
}
func RecordAuthFailure(peer, reason string) {
if authFailuresTotal == nil {
return
}
authFailuresTotal.Add(context.Background(), 1, observability.Labels{"peer": peer, "reason": reason})
}
func RecordACLDenied(ifname, peer, policy string) {
if aclDeniedTotal == nil {
return
}
aclDeniedTotal.Add(context.Background(), 1, observability.Labels{"ifname": ifname, "peer": peer, "policy": policy})
}
func RecordCertificateExpiry(certName, ifname string, days float64) {
if certificateExpiryDays == nil {
return
}
certificateExpiryDays.Record(context.Background(), days, observability.Labels{"cert_name": certName, "ifname": ifname})
}
func RecordUDPPacket(ifname, packetType, direction string) {
if udpPacketsTotal == nil {
return
}
udpPacketsTotal.Add(context.Background(), 1, observability.Labels{"ifname": ifname, "type": packetType, "direction": direction})
}
func RecordUDPPacketSize(ifname, packetType string, bytes float64) {
if udpPacketSizeBytes == nil {
return
}
udpPacketSizeBytes.Record(context.Background(), bytes, observability.Labels{"ifname": ifname, "type": packetType})
}
func RecordHolePunchEvent(ifname, result string) {
if holePunchEventsTotal == nil {
return
}
holePunchEventsTotal.Add(context.Background(), 1, observability.Labels{"ifname": ifname, "result": result})
}
func RecordProxyMapping(ifname string, delta int64) {
if proxyMappingActive == nil {
return
}
proxyMappingActive.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
}
func RecordSession(ifname string, delta int64) {
if activeSessions == nil {
return
}
activeSessions.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
}
func RecordUDPConnection(ifname string, delta int64) {
if relayUDPConnectionsActive == nil {
return
}
relayUDPConnectionsActive.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
sessionActive.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
}
func RecordSessionRebuilt(ifname string) {
if sessionRebuiltTotal == nil {
return
}
sessionRebuiltTotal.Add(context.Background(), 1, observability.Labels{"ifname": ifname})
}
func RecordCommPattern(ifname string, delta int64) {
if commPatternActive == nil {
return
}
commPatternActive.Add(context.Background(), delta, observability.Labels{"ifname": ifname})
}
func RecordProxyCleanupRemoved(ifname, component string, count int64) {
if proxyCleanupRemovedTotal == nil {
return
}
proxyCleanupRemovedTotal.Add(context.Background(), count, observability.Labels{"ifname": ifname, "component": component})
}
func RecordProxyConnectionError(ifname, errorType string) {
if proxyConnectionErrorsTotal == nil {
return
}
proxyConnectionErrorsTotal.Add(context.Background(), 1, observability.Labels{"ifname": ifname, "error_type": errorType})
}
func RecordProxyInitialMappings(ifname string, count int64) {
if proxyInitialMappingsTotal == nil {
return
}
proxyInitialMappingsTotal.Record(context.Background(), count, observability.Labels{"ifname": ifname})
}
func RecordProxyMappingUpdate(ifname string) {
if proxyMappingUpdatesTotal == nil {
return
}
proxyMappingUpdatesTotal.Add(context.Background(), 1, observability.Labels{"ifname": ifname})
}
func RecordProxyIdleCleanupDuration(ifname, component string, seconds float64) {
if proxyIdleCleanupDuration == nil {
return
}
proxyIdleCleanupDuration.Record(context.Background(), seconds, observability.Labels{"ifname": ifname, "component": component})
}
func RecordSNIConnection(result string) {
if sniConnectionsTotal == nil {
return
}
sniConnectionsTotal.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordSNIConnectionDuration(seconds float64) {
if sniConnectionDuration == nil {
return
}
sniConnectionDuration.Record(context.Background(), seconds, nil)
}
func RecordSNIActiveConnection(delta int64) {
if sniActiveConnections == nil {
return
}
sniActiveConnections.Add(context.Background(), delta, nil)
}
func RecordSNIRouteCacheHit(result string) {
if sniRouteCacheHitsTotal == nil {
return
}
sniRouteCacheHitsTotal.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordSNIRouteAPIRequest(result string) {
if sniRouteAPIRequestsTotal == nil {
return
}
sniRouteAPIRequestsTotal.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordSNIRouteAPILatency(seconds float64) {
if sniRouteAPILatency == nil {
return
}
sniRouteAPILatency.Record(context.Background(), seconds, nil)
}
func RecordSNILocalOverride(hit string) {
if sniLocalOverrideTotal == nil {
return
}
sniLocalOverrideTotal.Add(context.Background(), 1, observability.Labels{"hit": hit})
}
func RecordSNITrustedProxyEvent(event string) {
if sniTrustedProxyEventsTotal == nil {
return
}
sniTrustedProxyEventsTotal.Add(context.Background(), 1, observability.Labels{"event": event})
}
func RecordSNIProxyProtocolParseError() {
if sniProxyProtocolParseErrorsTotal == nil {
return
}
sniProxyProtocolParseErrorsTotal.Add(context.Background(), 1, nil)
}
func RecordSNIDataBytes(direction string, bytes int64) {
if sniDataBytesTotal == nil {
return
}
sniDataBytesTotal.Add(context.Background(), bytes, observability.Labels{"direction": direction})
}
func RecordSNITunnelTermination(reason string) {
if sniTunnelTerminationsTotal == nil {
return
}
sniTunnelTerminationsTotal.Add(context.Background(), 1, observability.Labels{"reason": reason})
}
func RecordHTTPRequest(endpoint, method, statusCode string) {
if httpRequestsTotal == nil {
return
}
httpRequestsTotal.Add(context.Background(), 1, observability.Labels{"endpoint": endpoint, "method": method, "status_code": statusCode})
}
func RecordHTTPRequestDuration(endpoint, method string, seconds float64) {
if httpRequestDuration == nil {
return
}
httpRequestDuration.Record(context.Background(), seconds, observability.Labels{"endpoint": endpoint, "method": method})
}
func RecordPeerOperation(operation, result string) {
if peerOperationsTotal == nil {
return
}
peerOperationsTotal.Add(context.Background(), 1, observability.Labels{"operation": operation, "result": result})
}
func RecordProxyMappingUpdateRequest(result string) {
if proxyMappingUpdateRequestsTotal == nil {
return
}
proxyMappingUpdateRequestsTotal.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordDestinationsUpdateRequest(result string) {
if destinationsUpdateRequestsTotal == nil {
return
}
destinationsUpdateRequestsTotal.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordRemoteConfigFetch(result string) {
if remoteConfigFetchesTotal == nil {
return
}
remoteConfigFetchesTotal.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordBandwidthReport(result string) {
if bandwidthReportsTotal == nil {
return
}
bandwidthReportsTotal.Add(context.Background(), 1, observability.Labels{"result": result})
}
func RecordPeerBandwidthBytes(peer, direction string, bytes int64) {
if peerBandwidthBytesTotal == nil {
return
}
peerBandwidthBytesTotal.Add(context.Background(), bytes, observability.Labels{"peer": peer, "direction": direction})
}
func RecordMemorySpike(severity string) {
if memorySpikeTotal == nil {
return
}
memorySpikeTotal.Add(context.Background(), 1, observability.Labels{"severity": severity})
}
func RecordHeapProfileWritten() {
if heapProfilesWrittenTotal == nil {
return
}
heapProfilesWrittenTotal.Add(context.Background(), 1, nil)
}

View File

@@ -89,9 +89,6 @@ func TestDefaultConfig(t *testing.T) {
}
func TestShutdownNoInit(t *testing.T) {
// Ensure a known clean global state before testing no-init shutdown behavior.
_ = metrics.Shutdown(context.Background())
// Shutdown without Initialize should not panic or error.
if err := metrics.Shutdown(context.Background()); err != nil {
t.Errorf("unexpected error: %v", err)
@@ -171,7 +168,6 @@ func TestRecordRelay(t *testing.T) {
body := scrape(t, h)
assertContains(t, body, "gerbil_udp_packets_total")
assertContains(t, body, "gerbil_proxy_mapping_active")
assertContains(t, body, "gerbil_active_sessions")
}
func TestRecordWireGuard(t *testing.T) {
@@ -220,10 +216,10 @@ func TestRecordNetlink(t *testing.T) {
metrics.RecordKernelModuleLoad("success")
metrics.RecordFirewallRuleApplied("success", "INPUT")
metrics.RecordActiveSession("wg0", 1)
metrics.RecordActiveProxyConnection(1)
metrics.RecordProxyRouteLookup("hit")
metrics.RecordProxyTLSHandshake(0.05)
metrics.RecordProxyBytesTransmitted("tx", 1024)
metrics.RecordActiveProxyConnection(exampleHostname, 1)
metrics.RecordProxyRouteLookup("hit", exampleHostname)
metrics.RecordProxyTLSHandshake(exampleHostname, 0.05)
metrics.RecordProxyBytesTransmitted(exampleHostname, "tx", 1024)
body := scrape(t, h)
assertContains(t, body, "gerbil_netlink_events_total")
assertContains(t, body, "gerbil_active_sessions")

View File

@@ -60,10 +60,6 @@ type OTelConfig struct {
// ExportInterval is how often metrics are pushed to the collector.
// Defaults to 60 s.
ExportInterval time.Duration
// Timeout bounds OTLP exporter construction calls.
// Defaults to 10 s.
Timeout time.Duration
}
// DefaultMetricsConfig returns a MetricsConfig with sensible defaults.
@@ -79,7 +75,6 @@ func DefaultMetricsConfig() MetricsConfig {
Endpoint: "localhost:4317",
Insecure: true,
ExportInterval: 60 * time.Second,
Timeout: 10 * time.Second,
},
ServiceName: "gerbil",
ServiceVersion: "1.0.0",
@@ -93,10 +88,8 @@ func (c *MetricsConfig) Validate() error {
}
switch c.Backend {
case "prometheus", "none":
case "prometheus", "none", "":
// valid
case "":
return fmt.Errorf("metrics: enabled requires a non-empty backend")
case "otel":
if c.OTel.Endpoint == "" {
return fmt.Errorf("metrics: backend=otel requires a non-empty OTel endpoint")
@@ -107,9 +100,6 @@ func (c *MetricsConfig) Validate() error {
if c.OTel.ExportInterval <= 0 {
return fmt.Errorf("metrics: otel export interval must be positive")
}
if c.OTel.Timeout <= 0 {
return fmt.Errorf("metrics: otel timeout must be positive")
}
default:
return fmt.Errorf("metrics: unknown backend %q (must be \"prometheus\", \"otel\", or \"none\")", c.Backend)
}

View File

@@ -43,20 +43,20 @@ type Histogram interface {
type Backend interface {
// NewCounter creates a counter metric.
// labelNames declares the set of label keys that will be passed at observation time.
NewCounter(name, desc string, labelNames ...string) (Counter, error)
NewCounter(name, desc string, labelNames ...string) Counter
// NewUpDownCounter creates an up-down counter metric.
NewUpDownCounter(name, desc string, labelNames ...string) (UpDownCounter, error)
NewUpDownCounter(name, desc string, labelNames ...string) UpDownCounter
// NewInt64Gauge creates an integer gauge metric.
NewInt64Gauge(name, desc string, labelNames ...string) (Int64Gauge, error)
NewInt64Gauge(name, desc string, labelNames ...string) Int64Gauge
// NewFloat64Gauge creates a float gauge metric.
NewFloat64Gauge(name, desc string, labelNames ...string) (Float64Gauge, error)
NewFloat64Gauge(name, desc string, labelNames ...string) Float64Gauge
// NewHistogram creates a histogram metric.
// buckets are the explicit upper-bound bucket boundaries.
NewHistogram(name, desc string, buckets []float64, labelNames ...string) (Histogram, error)
NewHistogram(name, desc string, buckets []float64, labelNames ...string) Histogram
// HTTPHandler returns the /metrics HTTP handler.
// Implementations that do not expose an HTTP endpoint return nil.
@@ -88,7 +88,6 @@ func New(cfg MetricsConfig) (Backend, error) {
Endpoint: cfg.OTel.Endpoint,
Insecure: cfg.OTel.Insecure,
ExportInterval: cfg.OTel.ExportInterval,
Timeout: cfg.OTel.Timeout,
ServiceName: cfg.ServiceName,
ServiceVersion: cfg.ServiceVersion,
DeploymentEnvironment: cfg.DeploymentEnvironment,
@@ -111,19 +110,19 @@ type promAdapter struct {
b *obsprom.Backend
}
func (a *promAdapter) NewCounter(name, desc string, labelNames ...string) (Counter, error) {
func (a *promAdapter) NewCounter(name, desc string, labelNames ...string) Counter {
return a.b.NewCounter(name, desc, labelNames...)
}
func (a *promAdapter) NewUpDownCounter(name, desc string, labelNames ...string) (UpDownCounter, error) {
func (a *promAdapter) NewUpDownCounter(name, desc string, labelNames ...string) UpDownCounter {
return a.b.NewUpDownCounter(name, desc, labelNames...)
}
func (a *promAdapter) NewInt64Gauge(name, desc string, labelNames ...string) (Int64Gauge, error) {
func (a *promAdapter) NewInt64Gauge(name, desc string, labelNames ...string) Int64Gauge {
return a.b.NewInt64Gauge(name, desc, labelNames...)
}
func (a *promAdapter) NewFloat64Gauge(name, desc string, labelNames ...string) (Float64Gauge, error) {
func (a *promAdapter) NewFloat64Gauge(name, desc string, labelNames ...string) Float64Gauge {
return a.b.NewFloat64Gauge(name, desc, labelNames...)
}
func (a *promAdapter) NewHistogram(name, desc string, buckets []float64, labelNames ...string) (Histogram, error) {
func (a *promAdapter) NewHistogram(name, desc string, buckets []float64, labelNames ...string) Histogram {
return a.b.NewHistogram(name, desc, buckets, labelNames...)
}
func (a *promAdapter) HTTPHandler() http.Handler { return a.b.HTTPHandler() }
@@ -134,19 +133,19 @@ type otelAdapter struct {
b *obsotel.Backend
}
func (a *otelAdapter) NewCounter(name, desc string, labelNames ...string) (Counter, error) {
func (a *otelAdapter) NewCounter(name, desc string, labelNames ...string) Counter {
return a.b.NewCounter(name, desc, labelNames...)
}
func (a *otelAdapter) NewUpDownCounter(name, desc string, labelNames ...string) (UpDownCounter, error) {
func (a *otelAdapter) NewUpDownCounter(name, desc string, labelNames ...string) UpDownCounter {
return a.b.NewUpDownCounter(name, desc, labelNames...)
}
func (a *otelAdapter) NewInt64Gauge(name, desc string, labelNames ...string) (Int64Gauge, error) {
func (a *otelAdapter) NewInt64Gauge(name, desc string, labelNames ...string) Int64Gauge {
return a.b.NewInt64Gauge(name, desc, labelNames...)
}
func (a *otelAdapter) NewFloat64Gauge(name, desc string, labelNames ...string) (Float64Gauge, error) {
func (a *otelAdapter) NewFloat64Gauge(name, desc string, labelNames ...string) Float64Gauge {
return a.b.NewFloat64Gauge(name, desc, labelNames...)
}
func (a *otelAdapter) NewHistogram(name, desc string, buckets []float64, labelNames ...string) (Histogram, error) {
func (a *otelAdapter) NewHistogram(name, desc string, buckets []float64, labelNames ...string) Histogram {
return a.b.NewHistogram(name, desc, buckets, labelNames...)
}
func (a *otelAdapter) HTTPHandler() http.Handler { return a.b.HTTPHandler() }

View File

@@ -2,8 +2,6 @@ package observability_test
import (
"context"
"net"
"os"
"testing"
"time"
@@ -41,19 +39,20 @@ func TestValidateValidConfigs(t *testing.T) {
}{
{name: "disabled", cfg: observability.MetricsConfig{Enabled: false}},
{name: "backend none", cfg: observability.MetricsConfig{Enabled: true, Backend: "none"}},
{name: "backend empty", cfg: observability.MetricsConfig{Enabled: true, Backend: ""}},
{name: "prometheus", cfg: observability.MetricsConfig{Enabled: true, Backend: "prometheus"}},
{
name: "otel grpc",
cfg: observability.MetricsConfig{
Enabled: true, Backend: "otel",
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: otelGRPCEndpoint, ExportInterval: 10 * time.Second, Timeout: 2 * time.Second},
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: otelGRPCEndpoint, ExportInterval: 10 * time.Second},
},
},
{
name: "otel http",
cfg: observability.MetricsConfig{
Enabled: true, Backend: "otel",
OTel: observability.OTelConfig{Protocol: "http", Endpoint: "localhost:4318", ExportInterval: 30 * time.Second, Timeout: 2 * time.Second},
OTel: observability.OTelConfig{Protocol: "http", Endpoint: "localhost:4318", ExportInterval: 30 * time.Second},
},
},
}
@@ -72,36 +71,25 @@ func TestValidateInvalidConfigs(t *testing.T) {
cfg observability.MetricsConfig
}{
{name: "unknown backend", cfg: observability.MetricsConfig{Enabled: true, Backend: "datadog"}},
{
name: "backend empty while enabled",
cfg: observability.MetricsConfig{Enabled: true, Backend: ""},
},
{
name: "otel missing endpoint",
cfg: observability.MetricsConfig{
Enabled: true, Backend: "otel",
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: "", ExportInterval: 10 * time.Second, Timeout: 2 * time.Second},
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: "", ExportInterval: 10 * time.Second},
},
},
{
name: "otel invalid protocol",
cfg: observability.MetricsConfig{
Enabled: true, Backend: "otel",
OTel: observability.OTelConfig{Protocol: "tcp", Endpoint: otelGRPCEndpoint, ExportInterval: 10 * time.Second, Timeout: 2 * time.Second},
OTel: observability.OTelConfig{Protocol: "tcp", Endpoint: otelGRPCEndpoint, ExportInterval: 10 * time.Second},
},
},
{
name: "otel zero interval",
cfg: observability.MetricsConfig{
Enabled: true, Backend: "otel",
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: otelGRPCEndpoint, ExportInterval: 0, Timeout: 2 * time.Second},
},
},
{
name: "otel zero timeout",
cfg: observability.MetricsConfig{
Enabled: true, Backend: "otel",
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: otelGRPCEndpoint, ExportInterval: 10 * time.Second, Timeout: 0},
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: otelGRPCEndpoint, ExportInterval: 0},
},
},
}
@@ -169,32 +157,11 @@ func TestPrometheusAdapterAllInstruments(t *testing.T) {
ctx := context.Background()
labels := observability.Labels{"k": "v"}
c, err := b.NewCounter("prom_adapter_counter_total", "desc", "k")
if err != nil {
t.Fatalf("NewCounter error: %v", err)
}
u, err := b.NewUpDownCounter("prom_adapter_updown", "desc", "k")
if err != nil {
t.Fatalf("NewUpDownCounter error: %v", err)
}
ig, err := b.NewInt64Gauge("prom_adapter_int_gauge", "desc", "k")
if err != nil {
t.Fatalf("NewInt64Gauge error: %v", err)
}
fg, err := b.NewFloat64Gauge("prom_adapter_float_gauge", "desc", "k")
if err != nil {
t.Fatalf("NewFloat64Gauge error: %v", err)
}
h, err := b.NewHistogram("prom_adapter_histogram", "desc", []float64{0.1, 1.0}, "k")
if err != nil {
t.Fatalf("NewHistogram error: %v", err)
}
c.Add(ctx, 1, labels)
u.Add(ctx, 2, labels)
ig.Record(ctx, 99, labels)
fg.Record(ctx, 1.23, labels)
h.Record(ctx, 0.5, labels)
b.NewCounter("prom_adapter_counter_total", "desc", "k").Add(ctx, 1, labels)
b.NewUpDownCounter("prom_adapter_updown", "desc", "k").Add(ctx, 2, labels)
b.NewInt64Gauge("prom_adapter_int_gauge", "desc", "k").Record(ctx, 99, labels)
b.NewFloat64Gauge("prom_adapter_float_gauge", "desc", "k").Record(ctx, 1.23, labels)
b.NewHistogram("prom_adapter_histogram", "desc", []float64{0.1, 1.0}, "k").Record(ctx, 0.5, labels)
if b.HTTPHandler() == nil {
t.Error("prometheus adapter HTTPHandler should not be nil")
@@ -205,20 +172,9 @@ func TestPrometheusAdapterAllInstruments(t *testing.T) {
}
func TestOtelAdapterAllInstruments(t *testing.T) {
if os.Getenv("SKIP_OTEL_INTEGRATION") != "" {
t.Skip("skipping OTel integration test because SKIP_OTEL_INTEGRATION is set")
}
dialTimeout := 300 * time.Millisecond
conn, err := net.DialTimeout("tcp", otelGRPCEndpoint, dialTimeout)
if err != nil {
t.Skipf("skipping OTel integration test; collector %s not reachable: %v", otelGRPCEndpoint, err)
}
_ = conn.Close()
b, err := observability.New(observability.MetricsConfig{
Enabled: true, Backend: "otel",
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: otelGRPCEndpoint, Insecure: true, ExportInterval: 100 * time.Millisecond, Timeout: 2 * time.Second},
OTel: observability.OTelConfig{Protocol: "grpc", Endpoint: otelGRPCEndpoint, Insecure: true, ExportInterval: 100 * time.Millisecond},
})
if err != nil {
t.Fatalf("failed to create otel backend: %v", err)
@@ -226,32 +182,11 @@ func TestOtelAdapterAllInstruments(t *testing.T) {
ctx := context.Background()
labels := observability.Labels{"k": "v"}
c, err := b.NewCounter("otel_adapter_counter_total", "desc", "k")
if err != nil {
t.Fatalf("NewCounter error: %v", err)
}
u, err := b.NewUpDownCounter("otel_adapter_updown", "desc", "k")
if err != nil {
t.Fatalf("NewUpDownCounter error: %v", err)
}
ig, err := b.NewInt64Gauge("otel_adapter_int_gauge", "desc", "k")
if err != nil {
t.Fatalf("NewInt64Gauge error: %v", err)
}
fg, err := b.NewFloat64Gauge("otel_adapter_float_gauge", "desc", "k")
if err != nil {
t.Fatalf("NewFloat64Gauge error: %v", err)
}
h, err := b.NewHistogram("otel_adapter_histogram", "desc", []float64{0.1, 1.0}, "k")
if err != nil {
t.Fatalf("NewHistogram error: %v", err)
}
c.Add(ctx, 1, labels)
u.Add(ctx, 2, labels)
ig.Record(ctx, 99, labels)
fg.Record(ctx, 1.23, labels)
h.Record(ctx, 0.5, labels)
b.NewCounter("otel_adapter_counter_total", "desc", "k").Add(ctx, 1, labels)
b.NewUpDownCounter("otel_adapter_updown", "desc", "k").Add(ctx, 2, labels)
b.NewInt64Gauge("otel_adapter_int_gauge", "desc", "k").Record(ctx, 99, labels)
b.NewFloat64Gauge("otel_adapter_float_gauge", "desc", "k").Record(ctx, 1.23, labels)
b.NewHistogram("otel_adapter_histogram", "desc", []float64{0.1, 1.0}, "k").Record(ctx, 0.5, labels)
if b.HTTPHandler() != nil {
t.Error("OTel adapter HTTPHandler should be nil")

View File

@@ -13,31 +13,38 @@ type NoopBackend struct{}
// Compile-time interface check.
var _ Backend = (*NoopBackend)(nil)
func (n *NoopBackend) NewCounter(_ string, _ string, _ ...string) (Counter, error) {
return noopCounter{}, nil
func (n *NoopBackend) NewCounter(_ string, _ string, _ ...string) Counter {
_ = n
return noopCounter{}
}
func (n *NoopBackend) NewUpDownCounter(_ string, _ string, _ ...string) (UpDownCounter, error) {
return noopUpDownCounter{}, nil
func (n *NoopBackend) NewUpDownCounter(_ string, _ string, _ ...string) UpDownCounter {
_ = n
return noopUpDownCounter{}
}
func (n *NoopBackend) NewInt64Gauge(_ string, _ string, _ ...string) (Int64Gauge, error) {
return noopInt64Gauge{}, nil
func (n *NoopBackend) NewInt64Gauge(_ string, _ string, _ ...string) Int64Gauge {
_ = n
return noopInt64Gauge{}
}
func (n *NoopBackend) NewFloat64Gauge(_ string, _ string, _ ...string) (Float64Gauge, error) {
return noopFloat64Gauge{}, nil
func (n *NoopBackend) NewFloat64Gauge(_ string, _ string, _ ...string) Float64Gauge {
_ = n
return noopFloat64Gauge{}
}
func (n *NoopBackend) NewHistogram(_ string, _ string, _ []float64, _ ...string) (Histogram, error) {
return noopHistogram{}, nil
func (n *NoopBackend) NewHistogram(_ string, _ string, _ []float64, _ ...string) Histogram {
_ = n
return noopHistogram{}
}
func (n *NoopBackend) HTTPHandler() http.Handler {
_ = n
return nil
}
func (n *NoopBackend) Shutdown(_ context.Context) error {
_ = n
return nil
}

View File

@@ -13,32 +13,32 @@ func TestNoopBackendAllInstruments(t *testing.T) {
ctx := context.Background()
labels := observability.Labels{"k": "v"}
t.Run("Counter", func(t *testing.T) {
c, _ := n.NewCounter("test_counter", "desc")
t.Run("Counter", func(_ *testing.T) {
c := n.NewCounter("test_counter", "desc")
c.Add(ctx, 1, labels)
c.Add(ctx, 0, nil)
})
t.Run("UpDownCounter", func(t *testing.T) {
u, _ := n.NewUpDownCounter("test_updown", "desc")
t.Run("UpDownCounter", func(_ *testing.T) {
u := n.NewUpDownCounter("test_updown", "desc")
u.Add(ctx, 1, labels)
u.Add(ctx, -1, nil)
})
t.Run("Int64Gauge", func(t *testing.T) {
g, _ := n.NewInt64Gauge("test_int64gauge", "desc")
t.Run("Int64Gauge", func(_ *testing.T) {
g := n.NewInt64Gauge("test_int64gauge", "desc")
g.Record(ctx, 42, labels)
g.Record(ctx, 0, nil)
})
t.Run("Float64Gauge", func(t *testing.T) {
g, _ := n.NewFloat64Gauge("test_float64gauge", "desc")
t.Run("Float64Gauge", func(_ *testing.T) {
g := n.NewFloat64Gauge("test_float64gauge", "desc")
g.Record(ctx, 3.14, labels)
g.Record(ctx, 0, nil)
})
t.Run("Histogram", func(t *testing.T) {
h, _ := n.NewHistogram("test_histogram", "desc", []float64{1, 5, 10})
t.Run("Histogram", func(_ *testing.T) {
h := n.NewHistogram("test_histogram", "desc", []float64{1, 5, 10})
h.Record(ctx, 2.5, labels)
h.Record(ctx, 0, nil)
})
@@ -56,47 +56,12 @@ func TestNoopBackendAllInstruments(t *testing.T) {
})
}
func TestNoopBackendLabelNames(t *testing.T) {
func TestNoopBackendLabelNames(_ *testing.T) {
// Verify that label names passed at creation time are accepted without panic.
n := &observability.NoopBackend{}
assertNoPanic := func(t *testing.T, constructor string, fn func()) {
t.Helper()
defer func() {
if r := recover(); r != nil {
t.Fatalf("%s panicked: %v", constructor, r)
}
}()
fn()
}
t.Run("NewCounter", func(t *testing.T) {
assertNoPanic(t, "NewCounter", func() {
_, _ = n.NewCounter("c", "d", "label1", "label2")
})
})
t.Run("NewUpDownCounter", func(t *testing.T) {
assertNoPanic(t, "NewUpDownCounter", func() {
_, _ = n.NewUpDownCounter("u", "d", "l1")
})
})
t.Run("NewInt64Gauge", func(t *testing.T) {
assertNoPanic(t, "NewInt64Gauge", func() {
_, _ = n.NewInt64Gauge("g1", "d", "l1", "l2", "l3")
})
})
t.Run("NewFloat64Gauge", func(t *testing.T) {
assertNoPanic(t, "NewFloat64Gauge", func() {
_, _ = n.NewFloat64Gauge("g2", "d")
})
})
t.Run("NewHistogram", func(t *testing.T) {
assertNoPanic(t, "NewHistogram", func() {
_, _ = n.NewHistogram("h", "d", []float64{0.1, 1.0}, "l1")
})
})
n.NewCounter("c", "d", "label1", "label2")
n.NewUpDownCounter("u", "d", "l1")
n.NewInt64Gauge("g1", "d", "l1", "l2", "l3")
n.NewFloat64Gauge("g2", "d")
n.NewHistogram("h", "d", []float64{0.1, 1.0}, "l1")
}

View File

@@ -9,10 +9,7 @@ package otel
import (
"context"
"fmt"
"log"
"net/http"
"regexp"
"strings"
"time"
"go.opentelemetry.io/otel/attribute"
@@ -20,8 +17,6 @@ import (
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
)
var metricLabelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
// Config holds OTel backend configuration.
type Config struct {
// Protocol is "grpc" (default) or "http".
@@ -36,9 +31,6 @@ type Config struct {
// ExportInterval is the period between pushes to the collector.
ExportInterval time.Duration
// Timeout bounds exporter construction calls.
Timeout time.Duration
ServiceName string
ServiceVersion string
DeploymentEnvironment string
@@ -65,15 +57,9 @@ func New(cfg Config) (*Backend, error) {
if cfg.Protocol == "" {
cfg.Protocol = "grpc"
}
if strings.TrimSpace(cfg.Endpoint) == "" {
return nil, fmt.Errorf("otel backend: empty cfg.Endpoint")
}
if cfg.ExportInterval <= 0 {
cfg.ExportInterval = 60 * time.Second
}
if cfg.Timeout <= 0 {
cfg.Timeout = 10 * time.Second
}
if cfg.ServiceName == "" {
cfg.ServiceName = "gerbil"
}
@@ -114,196 +100,111 @@ func (b *Backend) Shutdown(ctx context.Context) error {
}
// NewCounter creates an OTel Int64Counter.
func (b *Backend) NewCounter(name, desc string, labelNames ...string) (*Counter, error) {
normalizedLabelNames, err := validateLabelNames(labelNames)
if err != nil {
return nil, fmt.Errorf("otel: create counter %q: %w", name, err)
}
func (b *Backend) NewCounter(name, desc string, _ ...string) *Counter {
c, err := b.meter.Int64Counter(name, metric.WithDescription(desc))
if err != nil {
return nil, fmt.Errorf("otel: create counter %q: %w", name, err)
panic(fmt.Sprintf("otel: create counter %q: %v", name, err))
}
return &Counter{c: c, labelNames: normalizedLabelNames}, nil
return &Counter{c: c}
}
// NewUpDownCounter creates an OTel Int64UpDownCounter.
func (b *Backend) NewUpDownCounter(name, desc string, labelNames ...string) (*UpDownCounter, error) {
normalizedLabelNames, err := validateLabelNames(labelNames)
if err != nil {
return nil, fmt.Errorf("otel: create up-down counter %q: %w", name, err)
}
func (b *Backend) NewUpDownCounter(name, desc string, _ ...string) *UpDownCounter {
c, err := b.meter.Int64UpDownCounter(name, metric.WithDescription(desc))
if err != nil {
return nil, fmt.Errorf("otel: create up-down counter %q: %w", name, err)
panic(fmt.Sprintf("otel: create up-down counter %q: %v", name, err))
}
return &UpDownCounter{c: c, labelNames: normalizedLabelNames}, nil
return &UpDownCounter{c: c}
}
// NewInt64Gauge creates an OTel Int64Gauge.
func (b *Backend) NewInt64Gauge(name, desc string, labelNames ...string) (*Int64Gauge, error) {
normalizedLabelNames, err := validateLabelNames(labelNames)
if err != nil {
return nil, fmt.Errorf("otel: create int64 gauge %q: %w", name, err)
}
func (b *Backend) NewInt64Gauge(name, desc string, _ ...string) *Int64Gauge {
g, err := b.meter.Int64Gauge(name, metric.WithDescription(desc))
if err != nil {
return nil, fmt.Errorf("otel: create int64 gauge %q: %w", name, err)
panic(fmt.Sprintf("otel: create int64 gauge %q: %v", name, err))
}
return &Int64Gauge{g: g, labelNames: normalizedLabelNames}, nil
return &Int64Gauge{g: g}
}
// NewFloat64Gauge creates an OTel Float64Gauge.
func (b *Backend) NewFloat64Gauge(name, desc string, labelNames ...string) (*Float64Gauge, error) {
normalizedLabelNames, err := validateLabelNames(labelNames)
if err != nil {
return nil, fmt.Errorf("otel: create float64 gauge %q: %w", name, err)
}
func (b *Backend) NewFloat64Gauge(name, desc string, _ ...string) *Float64Gauge {
g, err := b.meter.Float64Gauge(name, metric.WithDescription(desc))
if err != nil {
return nil, fmt.Errorf("otel: create float64 gauge %q: %w", name, err)
panic(fmt.Sprintf("otel: create float64 gauge %q: %v", name, err))
}
return &Float64Gauge{g: g, labelNames: normalizedLabelNames}, nil
return &Float64Gauge{g: g}
}
// NewHistogram creates an OTel Float64Histogram with explicit bucket boundaries.
func (b *Backend) NewHistogram(name, desc string, buckets []float64, labelNames ...string) (*Histogram, error) {
normalizedLabelNames, err := validateLabelNames(labelNames)
if err != nil {
return nil, fmt.Errorf("otel: create histogram %q: %w", name, err)
}
func (b *Backend) NewHistogram(name, desc string, buckets []float64, _ ...string) *Histogram {
h, err := b.meter.Float64Histogram(name,
metric.WithDescription(desc),
metric.WithExplicitBucketBoundaries(buckets...),
)
if err != nil {
return nil, fmt.Errorf("otel: create histogram %q: %w", name, err)
panic(fmt.Sprintf("otel: create histogram %q: %v", name, err))
}
return &Histogram{h: h, labelNames: normalizedLabelNames}, nil
return &Histogram{h: h}
}
func validateLabelNames(labelNames []string) ([]string, error) {
if len(labelNames) == 0 {
return nil, nil
// labelsToAttrs converts a Labels map to OTel attribute key-value pairs.
func labelsToAttrs(labels map[string]string) []attribute.KeyValue {
if len(labels) == 0 {
return nil
}
normalized := make([]string, len(labelNames))
seen := make(map[string]struct{}, len(labelNames))
for i, name := range labelNames {
if !metricLabelNameRE.MatchString(name) {
return nil, fmt.Errorf("invalid label name %q", name)
}
if _, exists := seen[name]; exists {
return nil, fmt.Errorf("duplicate label name %q", name)
}
seen[name] = struct{}{}
normalized[i] = name
attrs := make([]attribute.KeyValue, 0, len(labels))
for k, v := range labels {
attrs = append(attrs, attribute.String(k, v))
}
return normalized, nil
}
func labelsToAttrs(labelNames []string, labels map[string]string) []attribute.KeyValue {
if len(labelNames) == 0 {
if len(labels) > 0 {
log.Printf("WARN: dropping otel metric sample due to unexpected labels: got=%v expected=none", labels)
return nil
}
return []attribute.KeyValue{}
}
attrs := make([]attribute.KeyValue, 0, len(labelNames))
for _, labelName := range labelNames {
attrs = append(attrs, attribute.String(labelName, labels[labelName]))
}
for got := range labels {
found := false
for _, expected := range labelNames {
if got == expected {
found = true
break
}
}
if !found {
log.Printf("WARN: dropping otel metric sample due to unexpected label key %q (expected=%v)", got, labelNames)
return nil
}
}
return attrs
}
// Counter wraps an OTel Int64Counter.
type Counter struct {
c metric.Int64Counter
labelNames []string
c metric.Int64Counter
}
// Add increments the counter by value.
func (c *Counter) Add(ctx context.Context, value int64, labels map[string]string) {
attrs := labelsToAttrs(c.labelNames, labels)
if attrs == nil {
return
}
c.c.Add(ctx, value, metric.WithAttributes(attrs...))
c.c.Add(ctx, value, metric.WithAttributes(labelsToAttrs(labels)...))
}
// UpDownCounter wraps an OTel Int64UpDownCounter.
type UpDownCounter struct {
c metric.Int64UpDownCounter
labelNames []string
c metric.Int64UpDownCounter
}
// Add adjusts the up-down counter by value.
func (u *UpDownCounter) Add(ctx context.Context, value int64, labels map[string]string) {
attrs := labelsToAttrs(u.labelNames, labels)
if attrs == nil {
return
}
u.c.Add(ctx, value, metric.WithAttributes(attrs...))
u.c.Add(ctx, value, metric.WithAttributes(labelsToAttrs(labels)...))
}
// Int64Gauge wraps an OTel Int64Gauge.
type Int64Gauge struct {
g metric.Int64Gauge
labelNames []string
g metric.Int64Gauge
}
// Record sets the gauge to value.
func (g *Int64Gauge) Record(ctx context.Context, value int64, labels map[string]string) {
attrs := labelsToAttrs(g.labelNames, labels)
if attrs == nil {
return
}
g.g.Record(ctx, value, metric.WithAttributes(attrs...))
g.g.Record(ctx, value, metric.WithAttributes(labelsToAttrs(labels)...))
}
// Float64Gauge wraps an OTel Float64Gauge.
type Float64Gauge struct {
g metric.Float64Gauge
labelNames []string
g metric.Float64Gauge
}
// Record sets the gauge to value.
func (g *Float64Gauge) Record(ctx context.Context, value float64, labels map[string]string) {
attrs := labelsToAttrs(g.labelNames, labels)
if attrs == nil {
return
}
g.g.Record(ctx, value, metric.WithAttributes(attrs...))
g.g.Record(ctx, value, metric.WithAttributes(labelsToAttrs(labels)...))
}
// Histogram wraps an OTel Float64Histogram.
type Histogram struct {
h metric.Float64Histogram
labelNames []string
h metric.Float64Histogram
}
// Record observes value in the histogram.
func (h *Histogram) Record(ctx context.Context, value float64, labels map[string]string) {
attrs := labelsToAttrs(h.labelNames, labels)
if attrs == nil {
return
}
h.h.Record(ctx, value, metric.WithAttributes(attrs...))
h.h.Record(ctx, value, metric.WithAttributes(labelsToAttrs(labels)...))
}

View File

@@ -55,10 +55,7 @@ func TestOtelBackendCounter(t *testing.T) {
b := newInMemoryBackend(t)
defer b.Shutdown(context.Background()) //nolint:errcheck
c, err := b.NewCounter("gerbil_test_counter_total", "test counter", "result")
if err != nil {
t.Fatalf("NewCounter returned error: %v", err)
}
c := b.NewCounter("gerbil_test_counter_total", "test counter", "result")
// Should not panic
c.Add(context.Background(), 1, map[string]string{"result": "ok"})
c.Add(context.Background(), 5, nil)
@@ -68,10 +65,7 @@ func TestOtelBackendUpDownCounter(t *testing.T) {
b := newInMemoryBackend(t)
defer b.Shutdown(context.Background()) //nolint:errcheck
u, err := b.NewUpDownCounter("gerbil_test_updown", "test updown", "state")
if err != nil {
t.Fatalf("NewUpDownCounter returned error: %v", err)
}
u := b.NewUpDownCounter("gerbil_test_updown", "test updown", "state")
u.Add(context.Background(), 3, map[string]string{"state": "active"})
u.Add(context.Background(), -1, map[string]string{"state": "active"})
}
@@ -80,10 +74,7 @@ func TestOtelBackendInt64Gauge(t *testing.T) {
b := newInMemoryBackend(t)
defer b.Shutdown(context.Background()) //nolint:errcheck
g, err := b.NewInt64Gauge("gerbil_test_int_gauge", "test gauge")
if err != nil {
t.Fatalf("NewInt64Gauge returned error: %v", err)
}
g := b.NewInt64Gauge("gerbil_test_int_gauge", "test gauge")
g.Record(context.Background(), 42, nil)
}
@@ -91,10 +82,7 @@ func TestOtelBackendFloat64Gauge(t *testing.T) {
b := newInMemoryBackend(t)
defer b.Shutdown(context.Background()) //nolint:errcheck
g, err := b.NewFloat64Gauge("gerbil_test_float_gauge", "test float gauge")
if err != nil {
t.Fatalf("NewFloat64Gauge returned error: %v", err)
}
g := b.NewFloat64Gauge("gerbil_test_float_gauge", "test float gauge")
g.Record(context.Background(), 3.14, nil)
}
@@ -102,11 +90,8 @@ func TestOtelBackendHistogram(t *testing.T) {
b := newInMemoryBackend(t)
defer b.Shutdown(context.Background()) //nolint:errcheck
h, err := b.NewHistogram("gerbil_test_duration_seconds", "test histogram",
h := b.NewHistogram("gerbil_test_duration_seconds", "test histogram",
[]float64{0.1, 0.5, 1.0}, "method")
if err != nil {
t.Fatalf("NewHistogram returned error: %v", err)
}
h.Record(context.Background(), 0.3, map[string]string{"method": "GET"})
}
@@ -154,22 +139,3 @@ func TestOtelBackendDeploymentEnvironment(t *testing.T) {
}
defer b.Shutdown(context.Background()) //nolint:errcheck
}
func TestOtelBackendRejectsInvalidLabelNames(t *testing.T) {
b := newInMemoryBackend(t)
defer b.Shutdown(context.Background()) //nolint:errcheck
t.Run("duplicate labels", func(t *testing.T) {
_, err := b.NewCounter("gerbil_test_invalid_labels_total", "test counter", "result", "result")
if err == nil {
t.Fatal("expected error for duplicate label names")
}
})
t.Run("invalid label name", func(t *testing.T) {
_, err := b.NewHistogram("gerbil_test_invalid_histogram", "test histogram", []float64{0.1, 1.0}, "status-code")
if err == nil {
t.Fatal("expected error for invalid label name")
}
})
}

View File

@@ -3,8 +3,6 @@ package otel
import (
"context"
"fmt"
"net/url"
"strings"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
@@ -13,10 +11,6 @@ import (
// newExporter creates the appropriate OTLP exporter based on cfg.Protocol.
func newExporter(ctx context.Context, cfg Config) (sdkmetric.Exporter, error) {
if strings.TrimSpace(cfg.Endpoint) == "" {
return nil, fmt.Errorf("otel: cfg.Endpoint is empty")
}
switch cfg.Protocol {
case "grpc", "":
return newGRPCExporter(ctx, cfg)
@@ -42,20 +36,8 @@ func newGRPCExporter(ctx context.Context, cfg Config) (sdkmetric.Exporter, error
}
func newHTTPExporter(ctx context.Context, cfg Config) (sdkmetric.Exporter, error) {
endpoint := strings.TrimSpace(cfg.Endpoint)
opts := make([]otlpmetrichttp.Option, 0, 3)
if strings.Contains(endpoint, "://") {
parsed, err := url.Parse(endpoint)
if err != nil {
return nil, fmt.Errorf("otlp http exporter: parse endpoint URL %q: %w", endpoint, err)
}
opts = append(opts, otlpmetrichttp.WithEndpointURL(parsed.String()))
} else {
opts = append(opts,
otlpmetrichttp.WithEndpoint(endpoint),
otlpmetrichttp.WithURLPath("/v1/metrics"),
)
opts := []otlpmetrichttp.Option{
otlpmetrichttp.WithEndpoint(cfg.Endpoint),
}
if cfg.Insecure {
opts = append(opts, otlpmetrichttp.WithInsecure())

View File

@@ -3,7 +3,7 @@ package otel
import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/sdk/resource"
semconv "go.opentelemetry.io/otel/semconv/v1.18.0"
semconv "go.opentelemetry.io/otel/semconv/v1.40.0"
)
// newResource builds an OTel resource for the Gerbil service.
@@ -15,11 +15,11 @@ func newResource(serviceName, serviceVersion, deploymentEnv string) (*resource.R
attrs = append(attrs, semconv.ServiceVersion(serviceVersion))
}
if deploymentEnv != "" {
attrs = append(attrs, semconv.DeploymentEnvironment(deploymentEnv))
attrs = append(attrs, semconv.DeploymentEnvironmentName(deploymentEnv))
}
return resource.Merge(
resource.Default(),
resource.NewSchemaless(attrs...),
resource.NewWithAttributes(semconv.SchemaURL, attrs...),
)
}

View File

@@ -7,7 +7,6 @@ package prometheus
import (
"context"
"log"
"net/http"
"github.com/prometheus/client_golang/prometheus"
@@ -31,10 +30,9 @@ type Config struct {
// in the backend-specific instrument types that implement the observability
// instrument interfaces.
type Backend struct {
cfg Config
registry *prometheus.Registry
handler http.Handler
droppedSamplesCounter prometheus.Counter
cfg Config
registry *prometheus.Registry
handler http.Handler
}
// New creates and initialises a Prometheus backend.
@@ -50,11 +48,6 @@ func New(cfg Config) (*Backend, error) {
}
registry := prometheus.NewRegistry()
droppedSamplesCounter := prometheus.NewCounter(prometheus.CounterOpts{
Name: "gerbil_dropped_metric_samples_total",
Help: "Total number of metric samples dropped due to invalid labels or unsupported label sets",
})
registry.MustRegister(droppedSamplesCounter)
// Include Go and process metrics by default.
includeGo := cfg.IncludeGoMetrics == nil || *cfg.IncludeGoMetrics
@@ -69,7 +62,7 @@ func New(cfg Config) (*Backend, error) {
EnableOpenMetrics: false,
})
return &Backend{cfg: cfg, registry: registry, handler: handler, droppedSamplesCounter: droppedSamplesCounter}, nil
return &Backend{cfg: cfg, registry: registry, handler: handler}, nil
}
// HTTPHandler returns the Prometheus /metrics HTTP handler.
@@ -85,107 +78,60 @@ func (b *Backend) Shutdown(_ context.Context) error {
}
// NewCounter creates a Prometheus CounterVec registered on the backend's registry.
func (b *Backend) NewCounter(name, desc string, labelNames ...string) (*Counter, error) {
func (b *Backend) NewCounter(name, desc string, labelNames ...string) *Counter {
vec := prometheus.NewCounterVec(prometheus.CounterOpts{
Name: name,
Help: desc,
}, labelNames)
if err := b.registry.Register(vec); err != nil {
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
existing, ok := are.ExistingCollector.(*prometheus.CounterVec)
if !ok {
return nil, err
}
return &Counter{vec: existing, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
}
return nil, err
}
return &Counter{vec: vec, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
b.registry.MustRegister(vec)
return &Counter{vec: vec}
}
// NewUpDownCounter creates a Prometheus GaugeVec (Prometheus gauges are
// bidirectional) registered on the backend's registry.
func (b *Backend) NewUpDownCounter(name, desc string, labelNames ...string) (*UpDownCounter, error) {
func (b *Backend) NewUpDownCounter(name, desc string, labelNames ...string) *UpDownCounter {
vec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: name,
Help: desc,
}, labelNames)
if err := b.registry.Register(vec); err != nil {
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
existing, ok := are.ExistingCollector.(*prometheus.GaugeVec)
if !ok {
return nil, err
}
return &UpDownCounter{vec: existing, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
}
return nil, err
}
return &UpDownCounter{vec: vec, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
b.registry.MustRegister(vec)
return &UpDownCounter{vec: vec}
}
// NewInt64Gauge creates a Prometheus GaugeVec registered on the backend's registry.
func (b *Backend) NewInt64Gauge(name, desc string, labelNames ...string) (*Int64Gauge, error) {
func (b *Backend) NewInt64Gauge(name, desc string, labelNames ...string) *Int64Gauge {
vec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: name,
Help: desc,
}, labelNames)
if err := b.registry.Register(vec); err != nil {
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
existing, ok := are.ExistingCollector.(*prometheus.GaugeVec)
if !ok {
return nil, err
}
return &Int64Gauge{vec: existing, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
}
return nil, err
}
return &Int64Gauge{vec: vec, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
b.registry.MustRegister(vec)
return &Int64Gauge{vec: vec}
}
// NewFloat64Gauge creates a Prometheus GaugeVec registered on the backend's registry.
func (b *Backend) NewFloat64Gauge(name, desc string, labelNames ...string) (*Float64Gauge, error) {
func (b *Backend) NewFloat64Gauge(name, desc string, labelNames ...string) *Float64Gauge {
vec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: name,
Help: desc,
}, labelNames)
if err := b.registry.Register(vec); err != nil {
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
existing, ok := are.ExistingCollector.(*prometheus.GaugeVec)
if !ok {
return nil, err
}
return &Float64Gauge{vec: existing, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
}
return nil, err
}
return &Float64Gauge{vec: vec, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
b.registry.MustRegister(vec)
return &Float64Gauge{vec: vec}
}
// NewHistogram creates a Prometheus HistogramVec registered on the backend's registry.
func (b *Backend) NewHistogram(name, desc string, buckets []float64, labelNames ...string) (*Histogram, error) {
func (b *Backend) NewHistogram(name, desc string, buckets []float64, labelNames ...string) *Histogram {
vec := prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: name,
Help: desc,
Buckets: buckets,
}, labelNames)
if err := b.registry.Register(vec); err != nil {
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
existing, ok := are.ExistingCollector.(*prometheus.HistogramVec)
if !ok {
return nil, err
}
return &Histogram{vec: existing, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
}
return nil, err
}
return &Histogram{vec: vec, labelNames: append([]string(nil), labelNames...), droppedSamplesCounter: b.droppedSamplesCounter}, nil
b.registry.MustRegister(vec)
return &Histogram{vec: vec}
}
// Counter is a native Prometheus counter instrument.
type Counter struct {
vec *prometheus.CounterVec
labelNames []string
droppedSamplesCounter prometheus.Counter
vec *prometheus.CounterVec
}
// Add increments the counter by value for the given labels.
@@ -193,118 +139,47 @@ type Counter struct {
// value must be non-negative. Negative values are ignored.
func (c *Counter) Add(_ context.Context, value int64, labels map[string]string) {
if value < 0 {
log.Printf("WARN: counter add called with negative value=%d labels=%v expected_labels=%v", value, labels, c.labelNames)
return
}
normalized, ok := normalizeLabels(c.labelNames, labels, c.droppedSamplesCounter)
if !ok {
return
}
defer guardMetricPanic("counter", c.labelNames, labels)
c.vec.With(normalized).Add(float64(value))
c.vec.With(prometheus.Labels(labels)).Add(float64(value))
}
// UpDownCounter is a native Prometheus gauge used as a bidirectional counter.
type UpDownCounter struct {
vec *prometheus.GaugeVec
labelNames []string
droppedSamplesCounter prometheus.Counter
vec *prometheus.GaugeVec
}
// Add adjusts the gauge by value for the given labels.
func (u *UpDownCounter) Add(_ context.Context, value int64, labels map[string]string) {
normalized, ok := normalizeLabels(u.labelNames, labels, u.droppedSamplesCounter)
if !ok {
return
}
defer guardMetricPanic("updown", u.labelNames, labels)
u.vec.With(normalized).Add(float64(value))
u.vec.With(prometheus.Labels(labels)).Add(float64(value))
}
// Int64Gauge is a native Prometheus gauge recording integer snapshot values.
type Int64Gauge struct {
vec *prometheus.GaugeVec
labelNames []string
droppedSamplesCounter prometheus.Counter
vec *prometheus.GaugeVec
}
// Record sets the gauge to value for the given labels.
func (g *Int64Gauge) Record(_ context.Context, value int64, labels map[string]string) {
normalized, ok := normalizeLabels(g.labelNames, labels, g.droppedSamplesCounter)
if !ok {
return
}
defer guardMetricPanic("int64-gauge", g.labelNames, labels)
g.vec.With(normalized).Set(float64(value))
g.vec.With(prometheus.Labels(labels)).Set(float64(value))
}
// Float64Gauge is a native Prometheus gauge recording float snapshot values.
type Float64Gauge struct {
vec *prometheus.GaugeVec
labelNames []string
droppedSamplesCounter prometheus.Counter
vec *prometheus.GaugeVec
}
// Record sets the gauge to value for the given labels.
func (g *Float64Gauge) Record(_ context.Context, value float64, labels map[string]string) {
normalized, ok := normalizeLabels(g.labelNames, labels, g.droppedSamplesCounter)
if !ok {
return
}
defer guardMetricPanic("float64-gauge", g.labelNames, labels)
g.vec.With(normalized).Set(value)
g.vec.With(prometheus.Labels(labels)).Set(value)
}
// Histogram is a native Prometheus histogram instrument.
type Histogram struct {
vec *prometheus.HistogramVec
labelNames []string
droppedSamplesCounter prometheus.Counter
vec *prometheus.HistogramVec
}
// Record observes value for the given labels.
func (h *Histogram) Record(_ context.Context, value float64, labels map[string]string) {
normalized, ok := normalizeLabels(h.labelNames, labels, h.droppedSamplesCounter)
if !ok {
return
}
defer guardMetricPanic("histogram", h.labelNames, labels)
h.vec.With(normalized).Observe(value)
}
func normalizeLabels(labelNames []string, labels map[string]string, droppedSamplesCounter prometheus.Counter) (prometheus.Labels, bool) {
if len(labelNames) == 0 {
if len(labels) > 0 {
if droppedSamplesCounter != nil {
droppedSamplesCounter.Inc()
}
log.Printf("WARN: dropping metric sample due to unexpected labels: got=%v expected=none", labels)
return nil, false
}
return nil, true
}
normalized := make(prometheus.Labels, len(labelNames))
for _, name := range labelNames {
normalized[name] = ""
}
for k, v := range labels {
if _, ok := normalized[k]; !ok {
if droppedSamplesCounter != nil {
droppedSamplesCounter.Inc()
}
log.Printf("WARN: dropping metric sample due to unexpected label key %q (expected=%v)", k, labelNames)
return nil, false
}
normalized[k] = v
}
return normalized, true
}
func guardMetricPanic(kind string, expected []string, labels map[string]string) {
if recovered := recover(); recovered != nil {
log.Printf("WARN: dropped %s metric sample due to label panic: expected=%v got=%v err=%v", kind, expected, labels, recovered)
}
h.vec.With(prometheus.Labels(labels)).Observe(value)
}

View File

@@ -36,10 +36,7 @@ func TestPrometheusBackendShutdown(t *testing.T) {
func TestPrometheusBackendCounter(t *testing.T) {
b := newTestBackend(t)
c, err := b.NewCounter("test_counter_total", "A test counter", "result")
if err != nil {
t.Fatalf("NewCounter returned error: %v", err)
}
c := b.NewCounter("test_counter_total", "A test counter", "result")
c.Add(context.Background(), 3, map[string]string{"result": "ok"})
body := scrapeMetrics(t, b)
@@ -48,10 +45,7 @@ func TestPrometheusBackendCounter(t *testing.T) {
func TestPrometheusBackendUpDownCounter(t *testing.T) {
b := newTestBackend(t)
u, err := b.NewUpDownCounter("test_gauge_total", "A test up-down counter", "state")
if err != nil {
t.Fatalf("NewUpDownCounter returned error: %v", err)
}
u := b.NewUpDownCounter("test_gauge_total", "A test up-down counter", "state")
u.Add(context.Background(), 5, map[string]string{"state": "active"})
u.Add(context.Background(), -2, map[string]string{"state": "active"})
@@ -61,10 +55,7 @@ func TestPrometheusBackendUpDownCounter(t *testing.T) {
func TestPrometheusBackendInt64Gauge(t *testing.T) {
b := newTestBackend(t)
g, err := b.NewInt64Gauge("test_int_gauge", "An integer gauge", "ifname")
if err != nil {
t.Fatalf("NewInt64Gauge returned error: %v", err)
}
g := b.NewInt64Gauge("test_int_gauge", "An integer gauge", "ifname")
g.Record(context.Background(), 42, map[string]string{"ifname": "wg0"})
body := scrapeMetrics(t, b)
@@ -73,10 +64,7 @@ func TestPrometheusBackendInt64Gauge(t *testing.T) {
func TestPrometheusBackendFloat64Gauge(t *testing.T) {
b := newTestBackend(t)
g, err := b.NewFloat64Gauge("test_float_gauge", "A float gauge", "cert")
if err != nil {
t.Fatalf("NewFloat64Gauge returned error: %v", err)
}
g := b.NewFloat64Gauge("test_float_gauge", "A float gauge", "cert")
g.Record(context.Background(), 7.5, map[string]string{"cert": "example.com"})
body := scrapeMetrics(t, b)
@@ -86,10 +74,7 @@ func TestPrometheusBackendFloat64Gauge(t *testing.T) {
func TestPrometheusBackendHistogram(t *testing.T) {
b := newTestBackend(t)
buckets := []float64{0.1, 0.5, 1.0, 5.0}
h, err := b.NewHistogram("test_duration_seconds", "A test histogram", buckets, "method")
if err != nil {
t.Fatalf("NewHistogram returned error: %v", err)
}
h := b.NewHistogram("test_duration_seconds", "A test histogram", buckets, "method")
h.Record(context.Background(), 0.3, map[string]string{"method": "GET"})
body := scrapeMetrics(t, b)
@@ -100,10 +85,7 @@ func TestPrometheusBackendHistogram(t *testing.T) {
func TestPrometheusBackendMultipleLabels(t *testing.T) {
b := newTestBackend(t)
c, err := b.NewCounter("multi_label_total", "Multi-label counter", "method", "route", "status_code")
if err != nil {
t.Fatalf("NewCounter returned error: %v", err)
}
c := b.NewCounter("multi_label_total", "Multi-label counter", "method", "route", "status_code")
c.Add(context.Background(), 1, map[string]string{
"method": "POST",
"route": "/api/peers",
@@ -140,29 +122,23 @@ func TestPrometheusBackendNoGoMetrics(t *testing.T) {
func TestPrometheusBackendNilLabels(t *testing.T) {
// Adding with nil labels should not panic (treated as empty map).
b := newTestBackend(t)
c, err := b.NewCounter("nil_labels_total", "counter with no labels")
if err != nil {
t.Fatalf("NewCounter returned error: %v", err)
}
c := b.NewCounter("nil_labels_total", "counter with no labels")
// nil labels with no label names declared should be safe
c.Add(context.Background(), 1, nil)
}
func TestPrometheusBackendConcurrentAdd(t *testing.T) {
b := newTestBackend(t)
c, err := b.NewCounter("concurrent_total", "concurrent counter", "worker")
if err != nil {
t.Fatalf("NewCounter returned error: %v", err)
}
c := b.NewCounter("concurrent_total", "concurrent counter", "worker")
done := make(chan struct{})
for i := 0; i < 10; i++ {
go func() {
go func(_ int) {
for j := 0; j < 100; j++ {
c.Add(context.Background(), 1, map[string]string{"worker": "w"})
}
done <- struct{}{}
}()
}(i)
}
for i := 0; i < 10; i++ {
<-done
@@ -172,40 +148,6 @@ func TestPrometheusBackendConcurrentAdd(t *testing.T) {
assertMetricPresent(t, body, `concurrent_total{worker="w"} 1000`)
}
func TestPrometheusBackendAlreadyRegisteredCounter(t *testing.T) {
b := newTestBackend(t)
c1, err := b.NewCounter("dupe_counter_total", "duplicate counter", "result")
if err != nil {
t.Fatalf("first NewCounter returned error: %v", err)
}
c2, err := b.NewCounter("dupe_counter_total", "duplicate counter", "result")
if err != nil {
t.Fatalf("second NewCounter returned error: %v", err)
}
c1.Add(context.Background(), 1, map[string]string{"result": "ok"})
c2.Add(context.Background(), 2, map[string]string{"result": "ok"})
body := scrapeMetrics(t, b)
assertMetricPresent(t, body, `dupe_counter_total{result="ok"} 3`)
}
func TestPrometheusBackendInvalidLabelsNoPanic(t *testing.T) {
b := newTestBackend(t)
c, err := b.NewCounter("invalid_labels_total", "invalid labels test", "result")
if err != nil {
t.Fatalf("NewCounter returned error: %v", err)
}
// Extra label key should be dropped and must not panic.
c.Add(context.Background(), 5, map[string]string{"result": "ok", "unexpected": "x"})
body := scrapeMetrics(t, b)
if strings.Contains(body, `invalid_labels_total{result="ok"}`) {
t.Error("invalid label sample should have been dropped")
}
}
// --- helpers ---
func scrapeMetrics(t *testing.T, b *obsprom.Backend) string {

287
main.go
View File

@@ -3,7 +3,6 @@ package main
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"flag"
@@ -12,12 +11,10 @@ import (
"log"
"net"
"net/http"
"net/http/httputil"
_ "net/http/pprof"
"os"
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"runtime/pprof"
"strconv"
@@ -49,7 +46,6 @@ var (
doTrafficShaping bool
bandwidthLimit string
ifbName string // IFB device name for ingress traffic shaping
disableFirewall bool
)
type WgConfig struct {
@@ -106,144 +102,6 @@ type UpdateDestinationsRequest struct {
Destinations []relay.PeerDestination `json:"destinations"`
}
// pangolinDestHeader carries the downstream host:port (reachable over the
// WireGuard interface) that a /router request should be rewritten to,
// optionally prefixed with a scheme (e.g. "https://100.96.128.1:443"). It is
// stripped before the request is forwarded.
const pangolinDestHeader = "p-dest-header"
// pangolinHostHeader optionally carries the Host header value that should be
// sent to the destination, when it differs from pangolinDestHeader (e.g. the
// target's configured IP/hostname rather than the WireGuard routing
// address). It is stripped before the request is forwarded. When absent,
// the destination from pangolinDestHeader is used as the Host header, same
// as before.
const pangolinHostHeader = "p-dest-host-header"
// splitDestHeader separates an optional "scheme://" prefix from a
// pangolinDestHeader value, defaulting to "http" when none is present.
func splitDestHeader(dest string) (scheme, host string) {
if s, rest, ok := strings.Cut(dest, "://"); ok {
return s, rest
}
return "http", dest
}
// hostname strips an optional ":port" suffix from a host header value.
func hostname(hostport string) string {
if h, _, err := net.SplitHostPort(hostport); err == nil {
return h
}
return hostport
}
// routerSNIContextKey carries the TLS ServerName (SNI) that
// routerTransport's DialTLSContext should present, since we dial the
// WireGuard destination IP but the remote end typically terminates TLS
// based on the original hostname (pangolinHostHeader), not that IP.
type routerSNIContextKey struct{}
// routerTransport is routerProxy's RoundTripper. It mirrors
// http.DefaultTransport except for TLS dials, where it sets the SNI from
// routerSNIContextKey instead of letting it default to the dial address
// (the WireGuard IP), which the destination's TLS termination won't have a
// matching certificate/route for.
var routerTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
serverName := hostname(addr)
if sni, ok := ctx.Value(routerSNIContextKey{}).(string); ok && sni != "" {
serverName = sni
}
dialer := &tls.Dialer{Config: &tls.Config{ServerName: serverName}}
return dialer.DialContext(ctx, network, addr)
},
}
// logDebugRequest dumps a request's destination, headers, and body at debug
// level for troubleshooting (e.g. verifying an auth header made it through
// the proxy chain intact). It reads and restores req.Body so the request can
// still be sent afterward. Header values (including secrets like API keys)
// are logged as-is - only intended to be enabled for local troubleshooting.
func logDebugRequest(label string, req *http.Request) {
var headerLines strings.Builder
for name, values := range req.Header {
for _, v := range values {
fmt.Fprintf(&headerLines, "\n %s: %s", name, v)
}
}
body := []byte("<empty>")
if req.Body != nil {
data, err := io.ReadAll(req.Body)
req.Body.Close()
if err != nil {
logger.Error("%s: failed to read body for logging: %v", label, err)
} else {
body = data
}
req.Body = io.NopCloser(bytes.NewReader(data))
}
logger.Debug("%s: %s %s://%s%s headers:%s\nbody: %s", label, req.Method, req.URL.Scheme, req.Host, req.URL.RequestURI(), headerLines.String(), body)
}
// routerProxy forwards /router/* requests from the Pangolin AI gateway to a
// destination on the WireGuard network, as named by pangolinDestHeader.
// Used for proxying AI chat completion requests (incl. streaming) to
// providers reachable only from a site.
var routerProxy = &httputil.ReverseProxy{
Rewrite: func(pr *httputil.ProxyRequest) {
scheme, dest := splitDestHeader(pr.In.Header.Get(pangolinDestHeader))
pr.Out.URL.Scheme = scheme
pr.Out.URL.Host = dest
pr.Out.URL.Path = strings.TrimPrefix(pr.In.URL.Path, "/router")
if !strings.HasPrefix(pr.Out.URL.Path, "/") {
pr.Out.URL.Path = "/" + pr.Out.URL.Path
}
pr.Out.URL.RawPath = ""
pr.Out.Host = dest
if hostOverride := pr.In.Header.Get(pangolinHostHeader); hostOverride != "" {
pr.Out.Host = hostOverride
ctx := context.WithValue(pr.Out.Context(), routerSNIContextKey{}, hostname(hostOverride))
pr.Out = pr.Out.WithContext(ctx)
}
pr.Out.Header.Del(pangolinDestHeader)
pr.Out.Header.Del(pangolinHostHeader)
logger.Debug("Router proxy: %s %s -> %s (Host: %s)", pr.In.Method, pr.In.URL.Path, pr.Out.URL.String(), pr.Out.Host)
logDebugRequest("Router outbound request", pr.Out)
},
Transport: routerTransport,
// Flush written bytes to the client immediately rather than buffering,
// which is required for SSE-based streaming chat completions.
FlushInterval: -1,
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
logger.Error("Router proxy error for %s: %v", r.URL.Path, err)
http.Error(w, "Bad gateway", http.StatusBadGateway)
},
}
func handleRouter(w http.ResponseWriter, r *http.Request) {
dest := r.Header.Get(pangolinDestHeader)
hostOverride := r.Header.Get(pangolinHostHeader)
logger.Debug("Router request received: %s %s dest=%s host=%s remote=%s", r.Method, r.URL.Path, dest, hostOverride, r.RemoteAddr)
if dest == "" {
http.Error(w, fmt.Sprintf("Missing %s header", pangolinDestHeader), http.StatusBadRequest)
return
}
_, host := splitDestHeader(dest)
if _, _, err := net.SplitHostPort(host); err != nil {
http.Error(w, "Invalid destination", http.StatusBadRequest)
return
}
routerProxy.ServeHTTP(w, r)
}
// httpMetricsMiddleware wraps HTTP handlers with metrics tracking
func httpMetricsMiddleware(endpoint string, handler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
@@ -273,16 +131,6 @@ func (w *responseWriterWrapper) WriteHeader(statusCode int) {
w.ResponseWriter.WriteHeader(statusCode)
}
// Unwrap exposes the underlying ResponseWriter so http.ResponseController
// (used by httputil.ReverseProxy's streaming Flush, and by Hijack/Push
// callers) can see through this wrapper to the real http.Flusher etc.
// Without this, ReverseProxy's flushes on /router/* silently no-op and
// streamed responses (e.g. SSE) get buffered until the response completes
// instead of being forwarded incrementally.
func (w *responseWriterWrapper) Unwrap() http.ResponseWriter {
return w.ResponseWriter
}
func parseLogLevel(level string) logger.LogLevel {
switch strings.ToUpper(level) {
case "DEBUG":
@@ -301,6 +149,8 @@ func parseLogLevel(level string) logger.LogLevel {
}
func main() {
go monitorMemory(1024 * 1024 * 512) // trigger if memory usage exceeds 512MB
var (
err error
wgconfig WgConfig
@@ -325,7 +175,6 @@ func main() {
otelMetricsEndpoint string
otelMetricsInsecure bool
otelMetricsExportInterval time.Duration
otelMetricsTimeout time.Duration
)
interfaceName = os.Getenv("INTERFACE")
@@ -346,7 +195,6 @@ func main() {
proxyProtocolStr := os.Getenv("PROXY_PROTOCOL")
doTrafficShapingStr := os.Getenv("DO_TRAFFIC_SHAPING")
bandwidthLimitStr := os.Getenv("BANDWIDTH_LIMIT")
disableFirewallStr := os.Getenv("DISABLE_FIREWALL")
// Read metrics env vars (defaults applied by DefaultMetricsConfig; these override defaults).
metricsEnabled = true // default
@@ -381,14 +229,6 @@ func main() {
log.Printf("WARN: invalid OTEL_METRICS_EXPORT_INTERVAL=%q: %v", v, err2)
}
}
otelMetricsTimeout = 10 * time.Second // default
if v := os.Getenv("OTEL_METRICS_TIMEOUT"); v != "" {
if d, err2 := time.ParseDuration(v); err2 == nil {
otelMetricsTimeout = d
} else {
log.Printf("WARN: invalid OTEL_METRICS_TIMEOUT=%q: %v", v, err2)
}
}
if interfaceName == "" {
flag.StringVar(&interfaceName, "interface", "wg0", "Name of the WireGuard interface")
@@ -467,13 +307,6 @@ func main() {
flag.BoolVar(&doTrafficShaping, "do-traffic-shaping", false, "Whether to set up traffic shaping rules for peers (requires tc command and root privileges)")
}
if disableFirewallStr != "" {
disableFirewall = strings.ToLower(disableFirewallStr) == "true"
}
if disableFirewallStr == "" {
flag.BoolVar(&disableFirewall, "disable-firewall", false, "Disable WireGuard firewall rules to allow all inbound traffic on the interface")
}
if bandwidthLimitStr != "" {
bandwidthLimit = bandwidthLimitStr
}
@@ -489,19 +322,9 @@ func main() {
flag.StringVar(&otelMetricsEndpoint, "otel-metrics-endpoint", otelMetricsEndpoint, "OTLP collector endpoint (e.g. localhost:4317)")
flag.BoolVar(&otelMetricsInsecure, "otel-metrics-insecure", otelMetricsInsecure, "Disable TLS for OTLP connection")
flag.DurationVar(&otelMetricsExportInterval, "otel-metrics-export-interval", otelMetricsExportInterval, "Interval between OTLP metric pushes")
flag.DurationVar(&otelMetricsTimeout, "otel-metrics-timeout", otelMetricsTimeout, "Timeout for OTLP exporter setup")
flag.Parse()
// Heap profiles are dumped into the configured config directory (the same
// directory as the config file, or /var/config as a fallback for remote-config
// setups) so they land on the volume the operator actually has mounted.
heapDir := "/var/config/heap"
if configFile != "" {
heapDir = filepath.Join(filepath.Dir(configFile), "heap")
}
go monitorMemory(1024*1024*512, heapDir) // trigger if memory usage exceeds 512MB
// Derive IFB device name from the WireGuard interface name (Linux limit: 15 chars)
ifbName = "ifb_" + interfaceName
if len(ifbName) > 15 {
@@ -524,7 +347,6 @@ func main() {
Endpoint: otelMetricsEndpoint,
Insecure: otelMetricsInsecure,
ExportInterval: otelMetricsExportInterval,
Timeout: otelMetricsTimeout,
},
ServiceName: "gerbil",
ServiceVersion: "1.0.0",
@@ -579,12 +401,9 @@ func main() {
logger.Fatal("You must provide either a config file or a remote config URL, not both")
}
// clean up the remote config URL for backwards compatibility
remoteConfigURL = strings.TrimRight(remoteConfigURL, "/")
// clean up the reomte config URL for backwards compatibility
remoteConfigURL = strings.TrimSuffix(remoteConfigURL, "/gerbil/get-config")
remoteConfigURL = strings.TrimSuffix(remoteConfigURL, "/gerbil/receive-bandwidth")
remoteConfigURL = strings.TrimSuffix(remoteConfigURL, "/gerbil")
remoteConfigURL = strings.TrimRight(remoteConfigURL, "/")
remoteConfigURL = strings.TrimSuffix(remoteConfigURL, "/")
var key wgtypes.Key
// if generateAndSaveKeyTo is provided, generate a private key and save it to the file. if the file already exists, load the key from the file
@@ -721,12 +540,9 @@ func main() {
http.HandleFunc("/update-destinations", httpMetricsMiddleware("update_destinations", handleUpdateDestinations))
http.HandleFunc("/update-local-snis", httpMetricsMiddleware("update_local_snis", handleUpdateLocalSNIs))
http.HandleFunc("/healthz", httpMetricsMiddleware("healthz", handleHealthz))
http.HandleFunc("/router/", httpMetricsMiddleware("router", handleRouter))
// Register metrics endpoint only for Prometheus backend.
// OTel backend pushes to a collector; no /metrics endpoint needed.
// Note: metricsPath is registered directly without httpMetricsMiddleware to prevent infinite recursion.
// The metricsHandler must not be wrapped by the middleware, as it would observe its own observation calls.
if metricsHandler != nil {
http.Handle(metricsPath, metricsHandler)
logger.Info("Metrics endpoint enabled at %s", metricsPath)
@@ -903,9 +719,7 @@ func ensureWireguardInterface(wgconfig WgConfig) error {
logger.Warn("Failed to ensure MSS clamping: %v", err)
}
if disableFirewall {
logger.Warn("Firewall disabled: all inbound traffic on %s will be allowed", interfaceName)
} else if err := ensureWireguardFirewall(wgconfig.IpAddress); err != nil {
if err := ensureWireguardFirewall(); err != nil {
logger.Warn("Failed to ensure WireGuard firewall rules: %v", err)
}
@@ -1081,18 +895,11 @@ func ensureMSSClamping() error {
return nil
}
func ensureWireguardFirewall(localIpAddress string) error {
func ensureWireguardFirewall() error {
// Rules to enforce:
// 1. Allow established/related connections (responses to our outbound traffic)
// 2. Allow ICMP ping packets
// 3. Allow inbound traffic to ports 80/443 on the local IP only (for Traefik)
// 4. Drop all other inbound traffic from peers
// Strip any CIDR suffix so we're left with just the host IP
localIp := localIpAddress
if ip, _, err := net.ParseCIDR(localIpAddress); err == nil {
localIp = ip.String()
}
// 3. Drop all other inbound traffic from peers
// Define the rules we want to ensure exist
rules := [][]string{
@@ -1112,24 +919,6 @@ func ensureWireguardFirewall(localIpAddress string) error {
"--icmp-type", "8",
"-j", "ACCEPT",
},
// Allow inbound HTTP to the local IP only (for Traefik)
{
"-A", "INPUT",
"-i", interfaceName,
"-p", "tcp",
"--dport", "80",
"-d", localIp,
"-j", "ACCEPT",
},
// Allow inbound HTTPS to the local IP only (for Traefik)
{
"-A", "INPUT",
"-i", interfaceName,
"-p", "tcp",
"--dport", "443",
"-d", localIp,
"-j", "ACCEPT",
},
// Drop all other inbound traffic from WireGuard interface
{
"-A", "INPUT",
@@ -1373,12 +1162,10 @@ func removePeerInternal(publicKey string) error {
// Get current peer info before removing to clear relay connections and bandwidth limits
var wgIPs []string
allowedIPsCount := 0
device, err := wgClient.Device(interfaceName)
if err == nil {
for _, peer := range device.Peers {
if peer.PublicKey.String() == publicKey {
allowedIPsCount = len(peer.AllowedIPs)
// Extract WireGuard IPs from this peer's allowed IPs
for _, allowedIP := range peer.AllowedIPs {
wgIPs = append(wgIPs, allowedIP.IP.String())
@@ -1421,7 +1208,7 @@ func removePeerInternal(publicKey string) error {
// Record metrics
metrics.RecordPeersTotal(interfaceName, -1)
metrics.RecordAllowedIPsCount(interfaceName, publicKey, -int64(allowedIPsCount))
metrics.RecordAllowedIPsCount(interfaceName, publicKey, -int64(len(wgIPs)))
return nil
}
@@ -1691,24 +1478,6 @@ func calculatePeerBandwidth() ([]PeerBandwidth, error) {
return peerBandwidths, nil
}
// defaultBandwidthReportBatchSize caps how many peer bandwidth readings are
// sent in a single POST. Reporting every peer in one request grows unbounded
// with fleet size and can exceed the remote server's request body limit,
// which surfaces as "413 Payload Too Large" and silently drops that whole
// report cycle. Overridable via GERBIL_BANDWIDTH_BATCH_SIZE.
const defaultBandwidthReportBatchSize = 250
var bandwidthReportBatchSize = loadBandwidthReportBatchSize()
func loadBandwidthReportBatchSize() int {
if v := os.Getenv("GERBIL_BANDWIDTH_BATCH_SIZE"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return defaultBandwidthReportBatchSize
}
func reportPeerBandwidth(apiURL string) error {
bandwidths, err := calculatePeerBandwidth()
if err != nil {
@@ -1717,47 +1486,26 @@ func reportPeerBandwidth(apiURL string) error {
return fmt.Errorf("failed to calculate peer bandwidth: %v", err)
}
if len(bandwidths) == 0 {
return nil
}
var batchErrs []error
for start := 0; start < len(bandwidths); start += bandwidthReportBatchSize {
end := start + bandwidthReportBatchSize
if end > len(bandwidths) {
end = len(bandwidths)
}
if err := sendPeerBandwidthBatch(apiURL, bandwidths[start:end]); err != nil {
metrics.RecordBandwidthReport("error")
batchErrs = append(batchErrs, err)
continue
}
metrics.RecordBandwidthReport("success")
}
if len(batchErrs) > 0 {
return fmt.Errorf("failed to report %d bandwidth batch(es): %w", len(batchErrs), errors.Join(batchErrs...))
}
return nil
}
func sendPeerBandwidthBatch(apiURL string, batch []PeerBandwidth) error {
jsonData, err := json.Marshal(batch)
jsonData, err := json.Marshal(bandwidths)
if err != nil {
metrics.RecordBandwidthReport("error")
return fmt.Errorf("failed to marshal bandwidth data: %v", err)
}
resp, err := http.Post(apiURL, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
metrics.RecordBandwidthReport("error")
return fmt.Errorf("failed to send bandwidth data: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
metrics.RecordBandwidthReport("error")
return fmt.Errorf("API returned non-OK status: %s", resp.Status)
}
// Record successful bandwidth report
metrics.RecordBandwidthReport("success")
return nil
}
@@ -1786,7 +1534,7 @@ func notifyPeerChange(action, publicKey string) {
}
}
func monitorMemory(limit uint64, heapDir string) {
func monitorMemory(limit uint64) {
var m runtime.MemStats
for {
runtime.ReadMemStats(&m)
@@ -1802,9 +1550,8 @@ func monitorMemory(limit uint64, heapDir string) {
// Record memory spike metric
metrics.RecordMemorySpike(severity)
if err := os.MkdirAll(heapDir, 0o755); err != nil {
log.Println("could not create heap profile directory:", err)
} else if f, err := os.Create(filepath.Join(heapDir, fmt.Sprintf("heap-spike-%d.pprof", time.Now().Unix()))); err != nil {
f, err := os.Create(fmt.Sprintf("/var/config/heap/heap-spike-%d.pprof", time.Now().Unix()))
if err != nil {
log.Println("could not create profile:", err)
} else {
pprof.WriteHeapProfile(f)

View File

@@ -11,58 +11,17 @@ import (
"log"
"net"
"net/http"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/fosrl/gerbil/internal/metrics"
"github.com/fosrl/gerbil/logger"
"github.com/patrickmn/go-cache"
"golang.org/x/sync/errgroup"
)
// defaultMaxSNIConnections caps the number of concurrent client connections
// the SNI proxy will accept. Without a cap, a burst of connections (a
// scanner sweep, a client reconnect storm) spawns unbounded goroutines each
// holding copy buffers and sockets, exhausting container memory faster than
// GC/backpressure can catch up.
// Sized conservatively since each connection now holds up to two pooled
// 32KB copy buffers once the buffer pool is actually honored (see
// bufferedReader/bufferedWriter below). Overridable via
// GERBIL_MAX_SNI_CONNECTIONS.
const defaultMaxSNIConnections = 4096
var maxSNIConnections = loadMaxSNIConnections()
func loadMaxSNIConnections() int64 {
if v := os.Getenv("GERBIL_MAX_SNI_CONNECTIONS"); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
return n
}
}
return defaultMaxSNIConnections
}
// defaultMaxSNIConnectionsPerIP caps concurrent connections from a single
// source IP, independent of the global maxSNIConnections budget. Without
// this, one noisy/misbehaving client (or a single scanning host) can
// consume the entire global budget and lock out every other customer
// sharing this proxy. Overridable via GERBIL_MAX_SNI_CONNECTIONS_PER_IP.
const defaultMaxSNIConnectionsPerIP = 256
var maxSNIConnectionsPerIP = loadMaxSNIConnectionsPerIP()
func loadMaxSNIConnectionsPerIP() int64 {
if v := os.Getenv("GERBIL_MAX_SNI_CONNECTIONS_PER_IP"); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
return n
}
}
return defaultMaxSNIConnectionsPerIP
}
// RouteRecord represents a routing configuration
type RouteRecord struct {
Hostname string
@@ -118,22 +77,12 @@ type SNIProxy struct {
// Buffer pool for connection piping
bufferPool *sync.Pool
// activeConnections tracks concurrent client connections so
// acceptConnections can enforce maxSNIConnections.
activeConnections atomic.Int64
// perIPConnections tracks concurrent connections per source IP (map[string]*atomic.Int64)
// so acceptConnections can enforce maxSNIConnectionsPerIP and stop one
// client from starving the rest. Entries are removed once a given IP's
// count returns to zero, so this stays bounded by currently-connected
// distinct IPs (itself bounded by maxSNIConnections) rather than growing
// with every IP ever seen.
perIPConnections sync.Map
}
type activeTunnel struct {
conns []net.Conn
ctx context.Context
cancel context.CancelFunc
count int // protected by activeTunnelsLock
}
// readOnlyConn is a wrapper for io.Reader that implements net.Conn
@@ -512,46 +461,8 @@ func (p *SNIProxy) acceptConnections() {
}
}
if p.activeConnections.Load() >= maxSNIConnections {
logger.Debug("Max concurrent SNI connections (%d) reached, rejecting connection from %s", maxSNIConnections, conn.RemoteAddr())
metrics.RecordSNIConnection("rejected_max_connections")
conn.Close()
continue
}
remoteHost, _, err := net.SplitHostPort(conn.RemoteAddr().String())
if err != nil {
remoteHost = conn.RemoteAddr().String()
}
counterVal, _ := p.perIPConnections.LoadOrStore(remoteHost, new(atomic.Int64))
perIPCounter := counterVal.(*atomic.Int64)
if perIPCounter.Load() >= maxSNIConnectionsPerIP {
logger.Debug("Max concurrent SNI connections per IP (%d) reached for %s, rejecting connection", maxSNIConnectionsPerIP, remoteHost)
metrics.RecordSNIConnection("rejected_per_ip_limit")
conn.Close()
continue
}
perIPCounter.Add(1)
p.activeConnections.Add(1)
metrics.RecordSNIActiveConnection(1)
p.wg.Add(1)
go func() {
defer func() {
if perIPCounter.Add(-1) == 0 {
// Best-effort cleanup: only remove the map entry if it
// still holds this exact counter (a concurrent new
// connection from the same IP may have already bumped
// it back up via LoadOrStore, or replaced it after a
// prior race). Undercounting in that narrow race window
// just means one connection isn't rate-limited briefly,
// never unbounded growth.
p.perIPConnections.CompareAndDelete(remoteHost, counterVal)
}
}()
p.handleConnection(conn)
}()
go p.handleConnection(conn)
}
}
@@ -599,10 +510,6 @@ func (p *SNIProxy) extractSNI(conn net.Conn) (string, io.Reader, error) {
func (p *SNIProxy) handleConnection(clientConn net.Conn) {
defer p.wg.Done()
defer clientConn.Close()
defer func() {
p.activeConnections.Add(-1)
metrics.RecordSNIActiveConnection(-1)
}()
metrics.RecordSNIConnection("accepted")
@@ -644,7 +551,7 @@ func (p *SNIProxy) handleConnection(clientConn net.Conn) {
logger.Debug("SNI extraction failed: %v", err)
return
}
metrics.RecordProxyTLSHandshake(time.Since(clientHelloStart).Seconds())
metrics.RecordProxyTLSHandshake(hostname, time.Since(clientHelloStart).Seconds())
if hostname == "" {
log.Println("No SNI hostname found")
@@ -692,8 +599,8 @@ func (p *SNIProxy) handleConnection(clientConn net.Conn) {
defer targetConn.Close()
logger.Debug("Connected to target: %s:%d", route.TargetHost, route.TargetPort)
metrics.RecordActiveProxyConnection(1)
defer metrics.RecordActiveProxyConnection(-1)
metrics.RecordActiveProxyConnection(hostname, 1)
defer metrics.RecordActiveProxyConnection(hostname, -1)
// Send PROXY protocol header if enabled
if p.proxyProtocol {
@@ -713,37 +620,32 @@ func (p *SNIProxy) handleConnection(clientConn net.Conn) {
}
}
// Track this tunnel by SNI
// Track this tunnel by SNI using context for cancellation
p.activeTunnelsLock.Lock()
tunnel, ok := p.activeTunnels[hostname]
if !ok {
tunnel = &activeTunnel{}
ctx, cancel := context.WithCancel(p.ctx)
tunnel = &activeTunnel{ctx: ctx, cancel: cancel}
p.activeTunnels[hostname] = tunnel
}
tunnel.conns = append(tunnel.conns, actualClientConn)
tunnel.count++
tunnelCtx := tunnel.ctx
p.activeTunnelsLock.Unlock()
defer func() {
// Remove this conn from active tunnels
p.activeTunnelsLock.Lock()
if tunnel, ok := p.activeTunnels[hostname]; ok {
newConns := make([]net.Conn, 0, len(tunnel.conns))
for _, c := range tunnel.conns {
if c != actualClientConn {
newConns = append(newConns, c)
}
}
if len(newConns) == 0 {
tunnel.count--
if tunnel.count == 0 {
tunnel.cancel()
if p.activeTunnels[hostname] == tunnel {
delete(p.activeTunnels, hostname)
} else {
tunnel.conns = newConns
}
}
p.activeTunnelsLock.Unlock()
}()
// Start bidirectional data transfer
p.pipe(hostname, actualClientConn, targetConn, clientReader)
// Start bidirectional data transfer with tunnel-level cancellation context.
p.pipe(tunnelCtx, hostname, actualClientConn, targetConn, clientReader)
}
// getRoute retrieves routing information for a hostname
@@ -751,7 +653,7 @@ func (p *SNIProxy) getRoute(hostname, clientAddr string) (*RouteRecord, error) {
// Check local overrides first
if _, isOverride := p.localOverrides[hostname]; isOverride {
logger.Debug("Local override matched for hostname: %s", hostname)
metrics.RecordProxyRouteLookup("local_override")
metrics.RecordProxyRouteLookup("local_override", hostname)
return &RouteRecord{
Hostname: hostname,
TargetHost: p.localProxyAddr,
@@ -764,7 +666,7 @@ func (p *SNIProxy) getRoute(hostname, clientAddr string) (*RouteRecord, error) {
_, isLocal := p.localSNIs[hostname]
p.localSNIsLock.RUnlock()
if isLocal {
metrics.RecordProxyRouteLookup("local")
metrics.RecordProxyRouteLookup("local", hostname)
return &RouteRecord{
Hostname: hostname,
TargetHost: p.localProxyAddr,
@@ -775,16 +677,16 @@ func (p *SNIProxy) getRoute(hostname, clientAddr string) (*RouteRecord, error) {
// Check cache first
if cached, found := p.cache.Get(hostname); found {
if cached == nil {
metrics.RecordProxyRouteLookup("cached_not_found")
metrics.RecordProxyRouteLookup("cached_not_found", hostname)
return nil, nil // Cached negative result
}
logger.Debug("Cache hit for hostname: %s", hostname)
metrics.RecordProxyRouteLookup("cache_hit")
metrics.RecordProxyRouteLookup("cache_hit", hostname)
return cached.(*RouteRecord), nil
}
logger.Debug("Cache miss for hostname: %s, querying API", hostname)
metrics.RecordProxyRouteLookup("cache_miss")
metrics.RecordProxyRouteLookup("cache_miss", hostname)
// Query API with timeout
ctx, cancel := context.WithTimeout(p.ctx, 5*time.Second)
@@ -889,79 +791,49 @@ func (p *SNIProxy) selectStickyEndpoint(clientAddr string, endpoints []string) s
return endpoints[index]
}
// bufferedReader hides any io.WriterTo the wrapped reader implements (e.g.
// io.MultiReader, used by peekClientHello to replay the buffered
// ClientHello ahead of the raw connection). Without this, io.CopyBuffer
// bypasses the caller-supplied buffer entirely and lets WriteTo drive its
// own, uncapped allocations - defeating the point of bufferPool.
type bufferedReader struct {
io.Reader
}
// bufferedWriter hides any io.ReaderFrom the wrapped writer implements (e.g.
// *net.TCPConn's splice/sendfile fast path). Without this, io.CopyBuffer
// bypasses the caller-supplied buffer here too, so each copy allocates and
// manages its own buffer regardless of what's pooled.
type bufferedWriter struct {
io.Writer
}
// pipe handles bidirectional data transfer between connections
func (p *SNIProxy) pipe(hostname string, clientConn, targetConn net.Conn, clientReader io.Reader) {
var wg sync.WaitGroup
wg.Add(2)
func (p *SNIProxy) pipe(ctx context.Context, hostname string, clientConn, targetConn net.Conn, clientReader io.Reader) {
g, gCtx := errgroup.WithContext(ctx)
// closeOnce ensures we only close connections once
var closeOnce sync.Once
closeConns := func() {
closeOnce.Do(func() {
// Close both connections to unblock any pending reads
clientConn.Close()
targetConn.Close()
})
}
// Close connections when context cancels to unblock io.Copy operations
context.AfterFunc(gCtx, func() {
clientConn.Close()
targetConn.Close()
})
// Copy data from client to target (using the buffered reader)
go func() {
defer wg.Done()
defer closeConns()
// Get buffer from pool and return when done
// Copy data from client to target (using buffered reader and pooled memory).
g.Go(func() error {
bufPtr := p.bufferPool.Get().(*[]byte)
defer func() {
// Clear buffer before returning to pool to prevent data leakage
clear(*bufPtr)
p.bufferPool.Put(bufPtr)
}()
bytesCopied, err := io.CopyBuffer(bufferedWriter{targetConn}, bufferedReader{clientReader}, *bufPtr)
metrics.RecordProxyBytesTransmitted("client_to_target", bytesCopied)
bytesCopied, err := io.CopyBuffer(targetConn, clientReader, *bufPtr)
metrics.RecordProxyBytesTransmitted(hostname, "client_to_target", bytesCopied)
if err != nil && err != io.EOF {
logger.Debug("Copy client->target error: %v", err)
}
}()
return err
})
// Copy data from target to client
go func() {
defer wg.Done()
defer closeConns()
// Get buffer from pool and return when done
g.Go(func() error {
bufPtr := p.bufferPool.Get().(*[]byte)
defer func() {
// Clear buffer before returning to pool to prevent data leakage
clear(*bufPtr)
p.bufferPool.Put(bufPtr)
}()
bytesCopied, err := io.CopyBuffer(bufferedWriter{clientConn}, bufferedReader{targetConn}, *bufPtr)
metrics.RecordProxyBytesTransmitted("target_to_client", bytesCopied)
bytesCopied, err := io.CopyBuffer(clientConn, targetConn, *bufPtr)
metrics.RecordProxyBytesTransmitted(hostname, "target_to_client", bytesCopied)
if err != nil && err != io.EOF {
logger.Debug("Copy target->client error: %v", err)
}
}()
return err
})
wg.Wait()
_ = g.Wait()
}
// GetCacheStats returns cache statistics
@@ -997,16 +869,14 @@ func (p *SNIProxy) UpdateLocalSNIs(fullDomains []string) {
logger.Debug("Updated local SNIs, added %d, removed %d", len(newSNIs), len(removed))
// Terminate tunnels for removed SNIs
// Terminate tunnels for removed SNIs via context cancellation
if len(removed) > 0 {
p.activeTunnelsLock.Lock()
for _, sni := range removed {
if tunnels, ok := p.activeTunnels[sni]; ok {
for _, conn := range tunnels.conns {
conn.Close()
}
if tunnel, ok := p.activeTunnels[sni]; ok {
tunnel.cancel()
delete(p.activeTunnels, sni)
logger.Debug("Closed tunnels for SNI target change: %s", sni)
logger.Debug("Cancelled tunnel context for SNI target change: %s", sni)
}
}
p.activeTunnelsLock.Unlock()

View File

@@ -1,7 +1,6 @@
package relay
import (
"bufio"
"bytes"
"context"
"encoding/binary"
@@ -10,11 +9,8 @@ import (
"io"
"net"
"net/http"
"os"
"runtime"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/fosrl/gerbil/internal/metrics"
@@ -63,31 +59,8 @@ type PeerDestination struct {
}
type DestinationConn struct {
conn *net.UDPConn
// lastUsed is unix nanoseconds, read/written via atomic ops since it's
// touched from packet workers and the response goroutine concurrently
// with no lock, and is also scanned for LRU eviction below.
lastUsed atomic.Int64
}
// defaultMaxUDPConnections caps the number of concurrent per-peer outbound
// UDP sockets the relay will keep open in s.connections. Without a cap, a
// burst of peer churn creates sockets faster than the 5-minute idle cleanup
// can reap them, exhausting the host's ephemeral port range or fd ulimit.
// That surfaces as "dial udp ...: resource temporarily unavailable" on every
// subsequent packet and pegs the CPU logging the flood (outage 2026-07-03,
// recurrence 2026-07-05). Overridable via GERBIL_MAX_UDP_CONNECTIONS.
const defaultMaxUDPConnections = 8192
var maxUDPConnections = loadMaxUDPConnections()
func loadMaxUDPConnections() int64 {
if v := os.Getenv("GERBIL_MAX_UDP_CONNECTIONS"); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
return n
}
}
return defaultMaxUDPConnections
conn *net.UDPConn
lastUsed time.Time
}
// Type for storing WireGuard handshake information
@@ -164,24 +137,6 @@ const (
WireGuardMessageTypeTransportData = 4
)
// cachedEndpointState holds the last-known endpoint fields used for change detection.
// Timestamp is intentionally excluded since it always changes.
type cachedEndpointState struct {
OlmID string
NewtID string
Token string
IP string
Port int
PublicKey string
}
// cachedEndpointEntry wraps cachedEndpointState with a timestamp so the cache
// can be expired after a short TTL even when the endpoint fields are unchanged.
type cachedEndpointEntry struct {
state cachedEndpointState
cachedAt time.Time
}
// --- End Types ---
// bufferPool allows reusing buffers to reduce allocations.
@@ -198,13 +153,10 @@ type UDPProxyServer struct {
conn *net.UDPConn
proxyMappings sync.Map // map[string]ProxyMapping where key is "ip:port"
connections sync.Map // map[string]*DestinationConn where key is destination "ip:port"
// connectionCount mirrors len(connections) without an O(n) sync.Map walk,
// so the cap check in getOrCreateConnection is cheap on the hot path.
connectionCount atomic.Int64
privateKey wgtypes.Key
packetChan chan Packet
ctx context.Context
cancel context.CancelFunc
privateKey wgtypes.Key
packetChan chan Packet
ctx context.Context
cancel context.CancelFunc
// Session tracking for WireGuard peers
// Key format: "senderIndex:receiverIndex"
@@ -220,12 +172,6 @@ type UDPProxyServer struct {
// Cache for resolved UDP addresses to avoid per-packet DNS lookups
// Key: "ip:port" string, Value: *net.UDPAddr
addrCache sync.Map
// lastEndpointCache stores the last-known endpoint state per client (key: olmId:newtId)
// used to skip redundant HTTP notifications when nothing has changed.
lastEndpointCache sync.Map
// notifyChan is the async queue for hole-punch endpoint notifications.
// Dedicated notifier workers drain this channel and perform the HTTP call.
notifyChan chan ClientEndpoint
// ReachableAt is the URL where this server can be reached
ReachableAt string
}
@@ -238,7 +184,6 @@ func NewUDPProxyServer(parentCtx context.Context, addr, serverURL string, privat
serverURL: serverURL,
privateKey: privateKey,
packetChan: make(chan Packet, 50000), // Increased from 1000 to handle high throughput
notifyChan: make(chan ClientEndpoint, 1000),
ReachableAt: reachableAt,
ctx: ctx,
cancel: cancel,
@@ -247,15 +192,10 @@ func NewUDPProxyServer(parentCtx context.Context, addr, serverURL string, privat
// Start sets up the UDP listener, worker pool, and begins reading packets.
func (s *UDPProxyServer) Start() error {
// Fetch initial mappings asynchronously so a large (potentially 100MB+)
// response does not block the UDP listener from coming up. Any packets
// arriving for unknown mappings before the load completes will simply
// log and be repopulated via the hole-punch path.
go func() {
if err := s.fetchInitialMappings(); err != nil {
logger.Error("Failed to fetch initial mappings: %v", err)
}
}()
// Fetch initial mappings.
if err := s.fetchInitialMappings(); err != nil {
return fmt.Errorf("failed to fetch initial mappings: %v", err)
}
udpAddr, err := net.ResolveUDPAddr("udp", s.addr)
if err != nil {
@@ -297,11 +237,6 @@ func (s *UDPProxyServer) Start() error {
// Start the hole punch rate limiter cleanup routine
go s.cleanupHolePunchRateLimiter()
// Start async endpoint notifier workers (HTTP calls off the hot path)
for i := 0; i < 5; i++ {
go s.endpointNotifierWorker()
}
return nil
}
@@ -440,43 +375,7 @@ func (s *UDPProxyServer) packetWorker() {
ClientPublicKey: msg.PublicKey,
}
logger.Debug("Created endpoint from packet remoteAddr %s: IP=%s, Port=%d", packet.remoteAddr.String(), endpoint.IP, endpoint.Port)
// Check if anything meaningful changed before queuing an HTTP notification.
// The cache expires after 2.5 s so the server always receives a fresh
// timestamp within its 5-second staleness window.
const endpointCacheTTL = 2500 * time.Millisecond
cacheKey := endpoint.OlmID + ":" + endpoint.NewtID
newState := cachedEndpointState{
OlmID: endpoint.OlmID,
NewtID: endpoint.NewtID,
Token: endpoint.Token,
IP: endpoint.IP,
Port: endpoint.Port,
PublicKey: endpoint.ClientPublicKey,
}
if cached, ok := s.lastEndpointCache.Load(cacheKey); ok {
entry := cached.(cachedEndpointEntry)
if entry.state == newState && time.Since(entry.cachedAt) < endpointCacheTTL {
// Endpoint unchanged and cache still fresh - skip the HTTP call.
logger.Debug("Endpoint unchanged for %s, skipping notification", cacheKey)
metrics.RecordHolePunchEvent(relayIfname, "deduplicated")
s.clearSessionsForIP(endpoint.IP)
metrics.RecordHolePunchEvent(relayIfname, "success")
bufferPool.Put(packet.data[:1500])
continue
}
}
s.lastEndpointCache.Store(cacheKey, cachedEndpointEntry{state: newState, cachedAt: time.Now()})
// Queue the notification asynchronously so the hot path is not blocked by HTTP.
select {
case s.notifyChan <- endpoint:
case <-s.ctx.Done():
// shutting down
default:
logger.Debug("Notification queue full, dropping hole punch notification for %s:%d", endpoint.IP, endpoint.Port)
metrics.RecordHolePunchEvent(relayIfname, "queue_full")
}
s.notifyServer(endpoint)
s.clearSessionsForIP(endpoint.IP) // Clear sessions for this IP to allow re-establishment
metrics.RecordHolePunchEvent(relayIfname, "success")
}
@@ -485,22 +384,6 @@ func (s *UDPProxyServer) packetWorker() {
}
}
// endpointNotifierWorker drains the notifyChan and performs the HTTP notification for each
// hole-punch endpoint. Running several of these keeps latency low even when the server is slow.
func (s *UDPProxyServer) endpointNotifierWorker() {
for {
select {
case endpoint, ok := <-s.notifyChan:
if !ok {
return
}
s.notifyServer(endpoint)
case <-s.ctx.Done():
return
}
}
}
// decryptMessage decrypts the message using the server's private key
func (s *UDPProxyServer) decryptMessage(encMsg EncryptedHolePunchMessage) ([]byte, error) {
// Parse the ephemeral public key
@@ -536,7 +419,6 @@ func (s *UDPProxyServer) decryptMessage(encMsg EncryptedHolePunchMessage) ([]byt
}
func (s *UDPProxyServer) fetchInitialMappings() error {
logger.Info("Requesting initial proxy mappings")
body := bytes.NewBuffer([]byte(fmt.Sprintf(`{"publicKey": "%s"}`, s.privateKey.PublicKey().String())))
resp, err := http.Post(s.serverURL+"/gerbil/get-all-relays", "application/json", body)
if err != nil {
@@ -548,82 +430,24 @@ func (s *UDPProxyServer) fetchInitialMappings() error {
return fmt.Errorf("server returned non-OK status: %d, body: %s",
resp.StatusCode, string(body))
}
logger.Info("Received initial mappings, streaming decode")
// Stream-decode the response instead of buffering the entire body
// (which can be 100MB+) and then re-walking it with json.Unmarshal.
// This both lowers peak memory and lets us start populating the
// sync.Map as entries arrive.
dec := json.NewDecoder(bufio.NewReaderSize(resp.Body, 1<<20))
// Expect opening '{' of the top-level object.
tok, err := dec.Token()
data, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read opening token: %v", err)
return fmt.Errorf("failed to read response body: %v", err)
}
if d, ok := tok.(json.Delim); !ok || d != '{' {
return fmt.Errorf("expected '{' at top level, got %v", tok)
logger.Info("Received initial mappings: %s", string(data))
var initialMappings InitialMappings
if err := json.Unmarshal(data, &initialMappings); err != nil {
return fmt.Errorf("failed to unmarshal initial mappings: %v", err)
}
count := 0
now := time.Now()
for dec.More() {
keyTok, err := dec.Token()
if err != nil {
return fmt.Errorf("failed to read top-level key: %v", err)
}
key, ok := keyTok.(string)
if !ok {
return fmt.Errorf("expected string key at top level, got %T", keyTok)
}
if key != "mappings" {
// Skip unknown top-level fields without materializing them.
var skip json.RawMessage
if err := dec.Decode(&skip); err != nil {
return fmt.Errorf("failed to skip field %q: %v", key, err)
}
continue
}
// Expect opening '{' of the mappings object.
tok, err := dec.Token()
if err != nil {
return fmt.Errorf("failed to read mappings open: %v", err)
}
if d, ok := tok.(json.Delim); !ok || d != '{' {
return fmt.Errorf("expected '{' for mappings, got %v", tok)
}
for dec.More() {
mapKeyTok, err := dec.Token()
if err != nil {
return fmt.Errorf("failed to read mapping key: %v", err)
}
mapKey, ok := mapKeyTok.(string)
if !ok {
return fmt.Errorf("expected string mapping key, got %T", mapKeyTok)
}
var mapping ProxyMapping
if err := dec.Decode(&mapping); err != nil {
return fmt.Errorf("failed to decode mapping %q: %v", mapKey, err)
}
mapping.LastUsed = now
s.proxyMappings.Store(mapKey, mapping)
count++
}
// Consume closing '}' of mappings object.
if _, err := dec.Token(); err != nil {
return fmt.Errorf("failed to read mappings close: %v", err)
}
// Store mappings in our sync.Map.
for key, mapping := range initialMappings.Mappings {
// Initialize LastUsed timestamp for initial mappings
mapping.LastUsed = time.Now()
s.proxyMappings.Store(key, mapping)
}
metrics.RecordProxyInitialMappings(relayIfname, int64(count))
metrics.RecordProxyMapping(relayIfname, int64(count))
logger.Info("Loaded %d initial proxy mappings", count)
metrics.RecordProxyInitialMappings(relayIfname, int64(len(initialMappings.Mappings)))
metrics.RecordProxyMapping(relayIfname, int64(len(initialMappings.Mappings)))
logger.Info("Loaded %d initial proxy mappings", len(initialMappings.Mappings))
return nil
}
@@ -891,17 +715,10 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
// Check if we have an existing connection
if conn, ok := s.connections.Load(key); ok {
destConn := conn.(*DestinationConn)
destConn.lastUsed.Store(time.Now().UnixNano())
destConn.lastUsed = time.Now()
return destConn.conn, nil
}
// Enforce a hard cap on concurrent sockets so a burst of peer churn can't
// exhaust the host's ephemeral ports/fds. Evict the least-recently-used
// connection to make room instead of growing unbounded.
if s.connectionCount.Load() >= maxUDPConnections {
s.evictLRUConnection()
}
// Create new connection
newConn, err := net.DialUDP("udp", nil, destAddr)
if err != nil {
@@ -909,17 +726,11 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
return nil, fmt.Errorf("failed to create UDP connection: %v", err)
}
destConn := &DestinationConn{conn: newConn}
destConn.lastUsed.Store(time.Now().UnixNano())
// Store the new connection. If another goroutine raced us and already
// created one for this key, close ours and use theirs instead.
if existing, loaded := s.connections.LoadOrStore(key, destConn); loaded {
newConn.Close()
return existing.(*DestinationConn).conn, nil
}
s.connectionCount.Add(1)
metrics.RecordUDPConnection(relayIfname, 1)
// Store the new connection
s.connections.Store(key, &DestinationConn{
conn: newConn,
lastUsed: time.Now(),
})
// Start a goroutine to handle responses
go s.handleResponses(newConn, destAddr, remoteAddr)
@@ -927,33 +738,6 @@ func (s *UDPProxyServer) getOrCreateConnection(destAddr *net.UDPAddr, remoteAddr
return newConn, nil
}
// evictLRUConnection closes and removes the least-recently-used destination
// connection so a new one can be created under the concurrent connection cap.
func (s *UDPProxyServer) evictLRUConnection() {
var oldestKey interface{}
var oldestConn *DestinationConn
var oldestTime int64
s.connections.Range(func(key, value interface{}) bool {
destConn := value.(*DestinationConn)
lu := destConn.lastUsed.Load()
if oldestKey == nil || lu < oldestTime {
oldestKey = key
oldestConn = destConn
oldestTime = lu
}
return true
})
if oldestKey != nil {
s.connections.Delete(oldestKey)
oldestConn.conn.Close()
s.connectionCount.Add(-1)
metrics.RecordUDPConnection(relayIfname, -1)
metrics.RecordProxyCleanupRemoved(relayIfname, "conn_evicted", 1)
}
}
func (s *UDPProxyServer) handleResponses(conn *net.UDPConn, destAddr *net.UDPAddr, remoteAddr *net.UDPAddr) {
buffer := make([]byte, 1500)
for {
@@ -1005,32 +789,22 @@ func (s *UDPProxyServer) handleResponses(conn *net.UDPConn, destAddr *net.UDPAdd
// Add a cleanup method to periodically remove idle connections
func (s *UDPProxyServer) cleanupIdleConnections() {
// Ticker interval and idle threshold were previously 5min/10min, meaning
// a socket could sit open for up to 15 minutes after going idle. Under a
// reconnect/churn burst that lag is enough to exhaust ephemeral ports
// before cleanup catches up, so both are tightened here.
ticker := time.NewTicker(1 * time.Minute)
ticker := time.NewTicker(5 * time.Minute)
defer ticker.Stop()
for {
select {
case <-ticker.C:
cleanupStart := time.Now()
now := time.Now().UnixNano()
removed := int64(0)
now := time.Now()
s.connections.Range(func(key, value interface{}) bool {
destConn := value.(*DestinationConn)
if now-destConn.lastUsed.Load() > int64(5*time.Minute) {
if now.Sub(destConn.lastUsed) > 10*time.Minute {
destConn.conn.Close()
s.connections.Delete(key)
removed++
metrics.RecordProxyCleanupRemoved(relayIfname, "conn", 1)
}
return true
})
if removed > 0 {
s.connectionCount.Add(-removed)
metrics.RecordUDPConnection(relayIfname, -removed)
metrics.RecordProxyCleanupRemoved(relayIfname, "conn", removed)
}
metrics.RecordProxyIdleCleanupDuration(relayIfname, "conn", time.Since(cleanupStart).Seconds())
case <-s.ctx.Done():
return
@@ -1192,10 +966,6 @@ func (s *UDPProxyServer) clearConnectionsForWGIP(wgIP string) {
for _, key := range keysToDelete {
s.connections.Delete(key)
}
if len(keysToDelete) > 0 {
s.connectionCount.Add(-int64(len(keysToDelete)))
metrics.RecordUDPConnection(relayIfname, -int64(len(keysToDelete)))
}
logger.Info("Cleared %d connections for WG IP: %s", len(keysToDelete), wgIP)
}
@@ -1393,56 +1163,33 @@ func (s *UDPProxyServer) trackCommunicationPattern(fromAddr, toAddr *net.UDPAddr
// tryRebuildSession attempts to rebuild a WireGuard session from communication patterns
func (s *UDPProxyServer) tryRebuildSession(pattern *CommunicationPattern) {
// Require both indices and a minimum amount of bidirectional traffic
if pattern.ClientIndex == 0 || pattern.DestIndex == 0 || pattern.PacketCount < 4 {
return
}
// Check if we have bidirectional communication within a reasonable time window
timeDiff := pattern.LastFromClient.Sub(pattern.LastFromDest)
if timeDiff < 0 {
timeDiff = -timeDiff
}
if timeDiff >= 30*time.Second {
return
}
sessionKey := fmt.Sprintf("%d:%d", pattern.DestIndex, pattern.ClientIndex)
destStr := pattern.ToDestination.String()
// Only rebuild if we have recent bidirectional communication and both indices
if timeDiff < 30*time.Second && pattern.ClientIndex != 0 && pattern.DestIndex != 0 && pattern.PacketCount >= 4 {
// Create session mapping: client's index maps to destination
sessionKey := fmt.Sprintf("%d:%d", pattern.DestIndex, pattern.ClientIndex)
// Fast path: if a matching session already exists, just refresh LastSeen and bail out.
// This prevents log spam and repeated work for every packet of an established flow.
if existing, ok := s.wgSessions.Load(sessionKey); ok {
sess := existing.(*WireGuardSession)
if da := sess.GetDestAddr(); da != nil && da.String() == destStr {
sess.UpdateLastSeen()
// Make sure the receiver-index fast-path is populated so future packets
// don't keep falling back to broadcast + pattern tracking.
if _, indexed := s.sessionsByReceiverIndex.Load(pattern.ClientIndex); !indexed {
s.sessionsByReceiverIndex.Store(pattern.ClientIndex, sess)
}
return
// Check if we already have this session
session := &WireGuardSession{
ReceiverIndex: pattern.DestIndex,
SenderIndex: pattern.ClientIndex,
DestAddr: pattern.ToDestination,
LastSeen: time.Now(),
}
if _, loaded := s.wgSessions.LoadOrStore(sessionKey, session); loaded {
s.wgSessions.Store(sessionKey, session)
} else {
metrics.RecordSession(relayIfname, 1)
metrics.RecordSessionRebuilt(relayIfname)
}
logger.Info("Rebuilt WireGuard session from communication pattern: %s -> %s (packets: %d)",
sessionKey, pattern.ToDestination.String(), pattern.PacketCount)
}
// Create or replace the session mapping
session := &WireGuardSession{
ReceiverIndex: pattern.DestIndex,
SenderIndex: pattern.ClientIndex,
DestAddr: pattern.ToDestination,
LastSeen: time.Now(),
}
if _, loaded := s.wgSessions.LoadOrStore(sessionKey, session); loaded {
s.wgSessions.Store(sessionKey, session)
} else {
metrics.RecordSession(relayIfname, 1)
metrics.RecordSessionRebuilt(relayIfname)
}
// Index by client receiver index so the transport-data fast path can find it.
s.sessionsByReceiverIndex.Store(pattern.ClientIndex, session)
logger.Info("Rebuilt WireGuard session from communication pattern: %s -> %s (packets: %d)",
sessionKey, destStr, pattern.PacketCount)
}
// cleanupIdleCommunicationPatterns periodically removes idle communication patterns