mirror of
https://github.com/netbirdio/netbird.git
synced 2026-09-24 23:59:08 +02:00
[management,signal,proxy] add pyroscope profiling (#7536)
This commit is contained in:
Pascal Fischer
@@ -0,0 +1,127 @@
|
||||
package profiling
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/caarlos0/env/v11"
|
||||
"github.com/grafana/pyroscope-go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var errNotConfigured = errors.New("pyroscope not configured")
|
||||
|
||||
var started atomic.Bool
|
||||
|
||||
type config struct {
|
||||
Address string `env:"NB_PYROSCOPE_ADDRESS"`
|
||||
User string `env:"NB_PYROSCOPE_USER,notEmpty"`
|
||||
Password string `env:"NB_PYROSCOPE_PASSWORD,notEmpty"`
|
||||
}
|
||||
|
||||
func Start(applicationName string) func() {
|
||||
noop := func() {}
|
||||
|
||||
cfg, err := loadConfig()
|
||||
switch {
|
||||
case errors.Is(err, errNotConfigured):
|
||||
log.Info("pyroscope not configured, continuous profiling disabled")
|
||||
return noop
|
||||
case err != nil:
|
||||
log.Errorf("failed to load pyroscope config: %v", err)
|
||||
return noop
|
||||
}
|
||||
|
||||
// pprof allows one CPU profile per process, so a second profiler (e.g. the
|
||||
// signal server inside the combined binary) would only log errors.
|
||||
if !started.CompareAndSwap(false, true) {
|
||||
log.Warnf("continuous profiling already running in this process, not starting it for %s", applicationName)
|
||||
return noop
|
||||
}
|
||||
|
||||
tags := map[string]string{}
|
||||
if hostname, err := os.Hostname(); err == nil {
|
||||
tags["instance"] = hostname
|
||||
} else {
|
||||
log.Warnf("failed to resolve hostname for profile tags: %v", err)
|
||||
}
|
||||
|
||||
profiler, err := pyroscope.Start(pyroscope.Config{
|
||||
ApplicationName: applicationName,
|
||||
ServerAddress: cfg.Address,
|
||||
BasicAuthUser: cfg.User,
|
||||
BasicAuthPassword: cfg.Password,
|
||||
Logger: log.StandardLogger(),
|
||||
Tags: tags,
|
||||
ProfileTypes: []pyroscope.ProfileType{
|
||||
pyroscope.ProfileCPU,
|
||||
pyroscope.ProfileAllocObjects,
|
||||
pyroscope.ProfileAllocSpace,
|
||||
pyroscope.ProfileInuseObjects,
|
||||
pyroscope.ProfileInuseSpace,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
started.Store(false)
|
||||
log.Errorf("failed to start continuous profiling: %v", err)
|
||||
return noop
|
||||
}
|
||||
|
||||
return func() {
|
||||
_ = profiler.Stop()
|
||||
started.Store(false)
|
||||
}
|
||||
}
|
||||
|
||||
func loadConfig() (config, error) {
|
||||
var cfg config
|
||||
if err := env.Parse(&cfg); err != nil {
|
||||
if cfg.Address == "" {
|
||||
return cfg, errNotConfigured
|
||||
}
|
||||
return cfg, fmt.Errorf("failed to parse pyroscope config: %w", err)
|
||||
}
|
||||
|
||||
if cfg.Address == "" {
|
||||
return cfg, errNotConfigured
|
||||
}
|
||||
if err := validateAddress(cfg.Address); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// validateAddress refuses to send the basic-auth credentials in plaintext to
|
||||
// anything but a loopback or private endpoint.
|
||||
func validateAddress(address string) error {
|
||||
u, err := url.Parse(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid pyroscope address %q: %w", address, err)
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "https":
|
||||
return nil
|
||||
case "http":
|
||||
if isLocalOrPrivate(u.Hostname()) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("insecure pyroscope address %q: use https for non-local endpoints", address)
|
||||
default:
|
||||
return fmt.Errorf("pyroscope address %q must use http or https", address)
|
||||
}
|
||||
}
|
||||
|
||||
func isLocalOrPrivate(host string) bool {
|
||||
if host == "localhost" || strings.HasSuffix(host, ".localhost") {
|
||||
return true
|
||||
}
|
||||
ip, err := netip.ParseAddr(host)
|
||||
return err == nil && (ip.IsLoopback() || ip.IsPrivate())
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package profiling
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
logtest "github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestStartSkipsSecondProfilerInProcess(t *testing.T) {
|
||||
clearEnv(t)
|
||||
t.Setenv("NB_PYROSCOPE_ADDRESS", "http://127.0.0.1:1")
|
||||
t.Setenv("NB_PYROSCOPE_USER", "user")
|
||||
t.Setenv("NB_PYROSCOPE_PASSWORD", "token")
|
||||
|
||||
started.Store(true)
|
||||
t.Cleanup(func() { started.Store(false) })
|
||||
hook := logtest.NewGlobal()
|
||||
t.Cleanup(hook.Reset)
|
||||
|
||||
stop := Start("netbird-second")
|
||||
stop()
|
||||
|
||||
assert.True(t, started.Load(), "the running profiler must stay marked as started")
|
||||
entry := hook.LastEntry()
|
||||
require.NotNil(t, entry, "the skipped start must be logged")
|
||||
assert.Equal(t, log.WarnLevel, entry.Level)
|
||||
assert.Contains(t, entry.Message, "already running")
|
||||
}
|
||||
|
||||
func TestLoadConfig(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
expected config
|
||||
errIs error
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "address unset disables profiling",
|
||||
errIs: errNotConfigured,
|
||||
},
|
||||
{
|
||||
name: "empty address disables profiling",
|
||||
env: map[string]string{"NB_PYROSCOPE_ADDRESS": ""},
|
||||
errIs: errNotConfigured,
|
||||
},
|
||||
{
|
||||
name: "credentials without address disable profiling",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
"NB_PYROSCOPE_PASSWORD": "token",
|
||||
},
|
||||
errIs: errNotConfigured,
|
||||
},
|
||||
{
|
||||
name: "address without credentials fails",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "https://profiles-prod-001.grafana.net",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "address with empty credentials fails",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "https://profiles-prod-001.grafana.net",
|
||||
"NB_PYROSCOPE_USER": "",
|
||||
"NB_PYROSCOPE_PASSWORD": "",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "address without password fails",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "https://profiles-prod-001.grafana.net",
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "full configuration",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "https://profiles-prod-001.grafana.net",
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
"NB_PYROSCOPE_PASSWORD": "token",
|
||||
},
|
||||
expected: config{
|
||||
Address: "https://profiles-prod-001.grafana.net",
|
||||
User: "123456",
|
||||
Password: "token",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "http to loopback is allowed",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "http://127.0.0.1:4040",
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
"NB_PYROSCOPE_PASSWORD": "token",
|
||||
},
|
||||
expected: config{
|
||||
Address: "http://127.0.0.1:4040",
|
||||
User: "123456",
|
||||
Password: "token",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "http to localhost is allowed",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "http://localhost:4040",
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
"NB_PYROSCOPE_PASSWORD": "token",
|
||||
},
|
||||
expected: config{
|
||||
Address: "http://localhost:4040",
|
||||
User: "123456",
|
||||
Password: "token",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "http to private network is allowed",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "http://10.0.0.5:4040",
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
"NB_PYROSCOPE_PASSWORD": "token",
|
||||
},
|
||||
expected: config{
|
||||
Address: "http://10.0.0.5:4040",
|
||||
User: "123456",
|
||||
Password: "token",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "http to public host is rejected",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "http://pyroscope.example.com",
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
"NB_PYROSCOPE_PASSWORD": "token",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "http to public address is rejected",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "http://203.0.113.10:4040",
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
"NB_PYROSCOPE_PASSWORD": "token",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "address without scheme is rejected",
|
||||
env: map[string]string{
|
||||
"NB_PYROSCOPE_ADDRESS": "pyroscope.example.com:4040",
|
||||
"NB_PYROSCOPE_USER": "123456",
|
||||
"NB_PYROSCOPE_PASSWORD": "token",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
clearEnv(t)
|
||||
for k, v := range tt.env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
|
||||
cfg, err := loadConfig()
|
||||
|
||||
switch {
|
||||
case tt.errIs != nil:
|
||||
require.ErrorIs(t, err, tt.errIs)
|
||||
case tt.wantErr:
|
||||
require.Error(t, err)
|
||||
require.NotErrorIs(t, err, errNotConfigured)
|
||||
default:
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expected, cfg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartWithoutConfigurationIsNoop(t *testing.T) {
|
||||
clearEnv(t)
|
||||
|
||||
stop := Start("netbird-test")
|
||||
require.NotNil(t, stop)
|
||||
stop()
|
||||
}
|
||||
|
||||
func clearEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
for _, k := range []string{"NB_PYROSCOPE_ADDRESS", "NB_PYROSCOPE_USER", "NB_PYROSCOPE_PASSWORD"} {
|
||||
t.Setenv(k, "")
|
||||
require.NoError(t, os.Unsetenv(k))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user