[management,signal,proxy,relay,misc] Unify service configuration loading

Service entry points currently resolve defaults, files, environment variables, and flags differently, which makes precedence inconsistent and prevents some services from using config files.

Introduce one Viper-backed loader and migrate Combined, Management, Relay, Signal, and Proxy while preserving compatibility aliases and Management template expansion.

util/config/loader.go

@@ -0,0 +1,384 @@
+// Package config loads service configuration from defaults, files, environment
+// variables, and command-line flags. Values are applied in that order, so an
+// explicitly changed flag has the highest precedence.
+//
+// Configuration keys come from the struct tag selected by [Options.TagName],
+// which defaults to "mapstructure". Environment variable names are inferred
+// from those keys with the NB prefix. For example, server.listen-address maps
+// to NB_SERVER_LISTEN_ADDRESS. The env and flag tags can provide explicit names,
+// comma-separated compatibility aliases, or "-" to disable a source.
+//
+// A typical service configuration can be loaded as follows:
+//
+//	type Config struct {
+//		Address string        `yaml:"address" env:"NB_ADDRESS" flag:"address"`
+//		Timeout time.Duration `yaml:"timeout"`
+//	}
+//
+//	flags := pflag.NewFlagSet("service", pflag.ContinueOnError)
+//	flags.String("address", ":443", "service listen address")
+//
+//	cfg, err := config.Load("config.yaml", &Config{
+//		Address: ":443",
+//		Timeout: 30 * time.Second,
+//	}, config.Options{
+//		TagName: "yaml",
+//		FlagSet: flags,
+//		Strict:  true,
+//	})
+//
+// Set [Options.AllowMissing] when the service must start without a configuration
+// file. [Options.Transform] can preprocess file contents before decoding, such
+// as with [ExpandEnvTemplate].
+package config
+
+import (
+	"bytes"
+	"encoding"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"reflect"
+	"slices"
+	"strings"
+
+	"github.com/go-viper/mapstructure/v2"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+)
+
+const envPrefix = "NB"
+
+var (
+	textUnmarshalerType = reflect.TypeFor[encoding.TextUnmarshaler]()
+	jsonUnmarshalerType = reflect.TypeFor[json.Unmarshaler]()
+)
+
+// Options controls how Load resolves files and fields.
+type Options struct {
+	// TagName selects the struct tag used for configuration keys.
+	TagName string
+	// AllowMissing permits an empty path or a file that does not exist.
+	AllowMissing bool
+	// FlagSet provides command-line flags referenced by `flag` struct tags.
+	FlagSet *pflag.FlagSet
+	// Transform rewrites configuration file contents before decoding.
+	Transform func([]byte) ([]byte, error)
+	// Strict rejects configuration keys that are not represented by the target type.
+	Strict bool
+}
+
+// Load reads configuration into a default-initialized value. Environment values
+// override file values, and file values override defaults.
+func Load[T any](configPath string, cfg *T, options Options) (*T, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("default config is nil")
+	}
+
+	configType := reflect.TypeFor[T]()
+	if configType.Kind() != reflect.Struct {
+		return nil, fmt.Errorf("config type %s must be a struct", configType)
+	}
+
+	if configPath == "" && !options.AllowMissing {
+		return nil, errors.New("config file path is required")
+	}
+
+	tagName := options.TagName
+	if tagName == "" {
+		tagName = "mapstructure"
+	}
+
+	configData, err := readConfigFile(configPath, options.AllowMissing)
+	if err != nil {
+		return nil, err
+	}
+
+	configFormat := ""
+	if configData != nil {
+		configFormat, err = resolveConfigType(configPath, tagName)
+		if err != nil {
+			return nil, err
+		}
+		if options.Transform != nil {
+			configData, err = options.Transform(configData)
+			if err != nil {
+				return nil, fmt.Errorf("transform config: %w", err)
+			}
+		}
+	}
+
+	v := viper.New()
+	if configFormat != "" {
+		v.SetConfigType(configFormat)
+	}
+	v.SetEnvPrefix(envPrefix)
+	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
+	v.AllowEmptyEnv(true)
+	v.AutomaticEnv()
+
+	if err := bindConfigSources(
+		v,
+		configType,
+		"",
+		tagName,
+		options.FlagSet,
+		true,
+		true,
+		make(map[reflect.Type]bool),
+	); err != nil {
+		return nil, fmt.Errorf("bind config sources: %w", err)
+	}
+
+	if configData != nil {
+		if err := v.ReadConfig(bytes.NewReader(configData)); err != nil {
+			return nil, fmt.Errorf("read config: %w", err)
+		}
+	}
+
+	var unmarshalErr error
+	if options.Strict {
+		unmarshalErr = v.UnmarshalExact(cfg, decoderConfig(tagName))
+	} else {
+		unmarshalErr = v.Unmarshal(cfg, decoderConfig(tagName))
+	}
+	if unmarshalErr != nil {
+		return nil, fmt.Errorf("unmarshal config: %w", unmarshalErr)
+	}
+	return cfg, nil
+}
+
+func readConfigFile(configPath string, allowMissing bool) ([]byte, error) {
+	if configPath == "" {
+		return nil, nil
+	}
+
+	data, err := os.ReadFile(configPath)
+	if err == nil {
+		return data, nil
+	}
+	if allowMissing && errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	}
+	return nil, fmt.Errorf("read config file: %w", err)
+}
+
+func resolveConfigType(configPath, fallbackType string) (string, error) {
+	extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(configPath)), ".")
+	if slices.Contains(viper.SupportedExts, extension) {
+		return extension, nil
+	}
+
+	fallbackType = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(fallbackType)), ".")
+	if fallbackType != "" {
+		if !slices.Contains(viper.SupportedExts, fallbackType) {
+			return "", fmt.Errorf("unsupported default config type %q", fallbackType)
+		}
+		return fallbackType, nil
+	}
+
+	if extension == "" {
+		return "", errors.New("config file extension is required")
+	}
+	return "", fmt.Errorf("unsupported config file extension %q", extension)
+}
+
+func decoderConfig(tagName string) viper.DecoderConfigOption {
+	return func(config *mapstructure.DecoderConfig) {
+		config.TagName = tagName
+		config.DecodeHook = mapstructure.ComposeDecodeHookFunc(
+			decodeLegacyBoolean,
+			mapstructure.TextUnmarshallerHookFunc(),
+			jsonUnmarshallerHook,
+			config.DecodeHook,
+		)
+	}
+}
+
+func bindConfigSources(
+	v *viper.Viper,
+	configType reflect.Type,
+	prefix string,
+	tagName string,
+	flagSet *pflag.FlagSet,
+	bindEnvironment bool,
+	bindFlags bool,
+	visiting map[reflect.Type]bool,
+) error {
+	for configType.Kind() == reflect.Pointer {
+		configType = configType.Elem()
+	}
+	visiting[configType] = true
+	defer delete(visiting, configType)
+
+	for i := range configType.NumField() {
+		field := configType.Field(i)
+		if !field.IsExported() {
+			continue
+		}
+
+		key, inline, skip := configFieldKey(field, tagName)
+		if skip {
+			continue
+		}
+		if inline {
+			key = prefix
+		} else if prefix != "" {
+			key = prefix + "." + key
+		}
+
+		fieldEnvironment := field.Tag.Get("env")
+		fieldFlags := field.Tag.Get("flag")
+		bindFieldEnvironment := bindEnvironment && fieldEnvironment != "-"
+		bindFieldFlags := bindFlags && fieldFlags != "-"
+
+		fieldType := field.Type
+		for fieldType.Kind() == reflect.Pointer {
+			fieldType = fieldType.Elem()
+		}
+		if fieldType.Kind() == reflect.Struct && !isScalarUnmarshaler(fieldType) {
+			if !visiting[fieldType] {
+				if err := bindConfigSources(
+					v,
+					fieldType,
+					key,
+					tagName,
+					flagSet,
+					bindFieldEnvironment,
+					bindFieldFlags,
+					visiting,
+				); err != nil {
+					return err
+				}
+			}
+			continue
+		}
+
+		if key == "" {
+			return fmt.Errorf("empty config key for field %s", field.Name)
+		}
+		if bindFieldEnvironment {
+			if err := bindEnvironmentVariable(v, key, fieldEnvironment); err != nil {
+				return err
+			}
+		}
+		if bindFieldFlags && flagSet != nil && fieldFlags != "" {
+			flagName, flag, err := selectFlag(flagSet, fieldFlags)
+			if err != nil {
+				return fmt.Errorf("config field %s: %w", field.Name, err)
+			}
+			if err := v.BindPFlag(key, flag); err != nil {
+				return fmt.Errorf("bind flag %s: %w", flagName, err)
+			}
+		}
+	}
+	return nil
+}
+
+func configFieldKey(field reflect.StructField, tagName string) (key string, inline, skip bool) {
+	tagParts := strings.Split(field.Tag.Get(tagName), ",")
+	key = tagParts[0]
+	if key == "-" {
+		return "", false, true
+	}
+	if key == "" {
+		key = field.Name
+	}
+	for _, option := range tagParts[1:] {
+		if option == "inline" || option == "squash" {
+			inline = true
+			break
+		}
+	}
+	return key, inline, false
+}
+
+func selectFlag(flagSet *pflag.FlagSet, names string) (string, *pflag.Flag, error) {
+	var selected *pflag.Flag
+	selectedName := ""
+	for _, name := range strings.Split(names, ",") {
+		flag := flagSet.Lookup(name)
+		if flag == nil {
+			return "", nil, fmt.Errorf("references unknown flag %q", name)
+		}
+		if selected == nil || flag.Changed {
+			selected = flag
+			selectedName = name
+		}
+		if flag.Changed {
+			break
+		}
+	}
+	return selectedName, selected, nil
+}
+
+func bindEnvironmentVariable(v *viper.Viper, key, environmentName string) error {
+	var err error
+	if environmentName == "" {
+		err = v.BindEnv(key)
+	} else {
+		names := strings.Split(environmentName, ",")
+		arguments := append([]string{key}, names...)
+		err = v.BindEnv(arguments...)
+	}
+	if err != nil {
+		return fmt.Errorf("bind environment for %s: %w", key, err)
+	}
+	return nil
+}
+
+func isScalarUnmarshaler(configType reflect.Type) bool {
+	return implements(configType, textUnmarshalerType) ||
+		implements(configType, jsonUnmarshalerType)
+}
+
+func implements(configType, interfaceType reflect.Type) bool {
+	return configType.Implements(interfaceType) ||
+		reflect.PointerTo(configType).Implements(interfaceType)
+}
+
+func jsonUnmarshallerHook(from, to reflect.Type, data any) (any, error) {
+	if !implements(to, jsonUnmarshalerType) {
+		return data, nil
+	}
+
+	raw, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+
+	targetType := to
+	if targetType.Kind() == reflect.Pointer {
+		targetType = targetType.Elem()
+	}
+	target := reflect.New(targetType)
+	unmarshaler, ok := target.Interface().(json.Unmarshaler)
+	if !ok {
+		return data, nil
+	}
+	if err := unmarshaler.UnmarshalJSON(raw); err != nil {
+		return nil, err
+	}
+	if to.Kind() == reflect.Pointer {
+		return target.Interface(), nil
+	}
+	return target.Elem().Interface(), nil
+}
+
+func decodeLegacyBoolean(from, to reflect.Kind, data any) (any, error) {
+	if from != reflect.String || to != reflect.Bool {
+		return data, nil
+	}
+
+	switch strings.ToLower(data.(string)) {
+	case "y", "yes", "on":
+		return true, nil
+	case "n", "no", "off":
+		return false, nil
+	default:
+		return data, nil
+	}
+}

util/config/loader_test.go

@@ -0,0 +1,238 @@
+package config
+
+import (
+	"encoding/json"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type testConfig struct {
+	Server   testServerConfig `yaml:"server"`
+	Internal string           `yaml:"-"`
+}
+
+type testServerConfig struct {
+	Address           string           `yaml:"address" env:"APP_SERVER_ADDRESS" flag:"address,legacy-address"`
+	BindAddress       netip.Addr       `yaml:"bindAddress"`
+	AdvertisedAddress netip.Addr       `yaml:"advertisedAddress"`
+	Enabled           bool             `yaml:"enabled"`
+	LogLevel          string           `yaml:"logLevel"`
+	Timeout           time.Duration    `yaml:"timeout"`
+	Ports             []int            `yaml:"ports"`
+	Owner             *testOwnerConfig `yaml:"owner,omitempty"`
+	TLS               testTLSConfig    `yaml:"tls"`
+	JSONValue         testJSONValue    `yaml:"jsonValue"`
+}
+
+type testOwnerConfig struct {
+	Email string `yaml:"email"`
+}
+
+type testTLSConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
+type testJSONValue struct {
+	Value string
+}
+
+func (v *testJSONValue) UnmarshalJSON(data []byte) error {
+	return json.Unmarshal(data, &v.Value)
+}
+
+type recursiveConfig struct {
+	Value string           `yaml:"value"`
+	Next  *recursiveConfig `yaml:"next,omitempty"`
+}
+
+func defaultTestConfig() *testConfig {
+	return &testConfig{
+		Server: testServerConfig{
+			Address:  ":443",
+			LogLevel: "info",
+			Timeout:  30 * time.Second,
+			Ports:    []int{443},
+		},
+		Internal: "default-internal",
+	}
+}
+
+func TestLoadAppliesFileEnvironmentAndDefaults(t *testing.T) {
+	configPath := writeConfigFile(t, "config.conf", `
+server:
+  bindAddress: 192.0.2.1
+  enabled: yes
+  logLevel: warn
+  timeout: 5s
+  ports: [80, 443]
+  jsonValue: decoded
+internal: ignored
+`)
+	t.Setenv("NB_SERVER_LOGLEVEL", "debug")
+	t.Setenv("NB_SERVER_OWNER_EMAIL", "owner@example.com")
+	t.Setenv("NB_SERVER_ADVERTISEDADDRESS", "198.51.100.1")
+
+	cfg, err := Load(configPath, defaultTestConfig(), Options{
+		TagName: "yaml",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, ":443", cfg.Server.Address, "Defaults should survive decoding")
+	assert.Equal(t, netip.MustParseAddr("192.0.2.1"), cfg.Server.BindAddress, "Text values from files should be decoded")
+	assert.Equal(t, netip.MustParseAddr("198.51.100.1"), cfg.Server.AdvertisedAddress, "Text values from the environment should be decoded")
+	assert.True(t, cfg.Server.Enabled, "Legacy YAML booleans should be decoded")
+	assert.Equal(t, "debug", cfg.Server.LogLevel, "Environment should override the file")
+	assert.Equal(t, 5*time.Second, cfg.Server.Timeout, "Durations should be decoded")
+	assert.Equal(t, []int{80, 443}, cfg.Server.Ports, "Slices should be decoded")
+	assert.Equal(t, "decoded", cfg.Server.JSONValue.Value, "JSON unmarshalers should be decoded")
+	require.NotNil(t, cfg.Server.Owner, "Environment should create optional nested configuration")
+	assert.Equal(t, "owner@example.com", cfg.Server.Owner.Email, "Nested environment values should be decoded")
+	assert.Equal(t, "default-internal", cfg.Internal, "Ignored fields should retain their defaults")
+}
+
+func TestLoadFlagPrecedence(t *testing.T) {
+	configPath := writeConfigFile(t, "config.yaml", `
+server:
+  address: ":8443"
+`)
+	t.Setenv("APP_SERVER_ADDRESS", ":9443")
+	flags := pflag.NewFlagSet("test", pflag.ContinueOnError)
+	flags.String("address", ":443", "")
+	flags.String("legacy-address", ":443", "")
+	require.NoError(t, flags.Set("legacy-address", ":7443"))
+
+	cfg, err := Load(configPath, defaultTestConfig(), Options{
+		TagName: "yaml",
+		FlagSet: flags,
+	})
+	require.NoError(t, err)
+	assert.Equal(t, ":7443", cfg.Server.Address, "Flags should override environment and file values")
+}
+
+func TestLoadAllowsEmptyEnvironmentOverrides(t *testing.T) {
+	configPath := writeConfigFile(t, "config.yaml", `
+server:
+  address: ":8443"
+`)
+	t.Setenv("APP_SERVER_ADDRESS", "")
+
+	cfg, err := Load(configPath, defaultTestConfig(), Options{
+		TagName: "yaml",
+	})
+	require.NoError(t, err)
+	assert.Empty(t, cfg.Server.Address, "An explicitly empty environment value should clear the file value")
+}
+
+func TestLoadTransformsConfig(t *testing.T) {
+	t.Setenv("CONFIG_ADDRESS", ":8443")
+	configPath := writeConfigFile(t, "config.yaml", `
+server:
+  address: "{{ .CONFIG_ADDRESS }}"
+`)
+
+	cfg, err := Load(configPath, defaultTestConfig(), Options{
+		TagName:   "yaml",
+		Transform: ExpandEnvTemplate,
+	})
+	require.NoError(t, err)
+	assert.Equal(t, ":8443", cfg.Server.Address, "The transform should run before decoding")
+}
+
+func TestLoadUsesRecognizedFileType(t *testing.T) {
+	configPath := writeConfigFile(t, "config.toml", `
+[server]
+address = ":8443"
+enabled = true
+`)
+
+	cfg, err := Load(configPath, defaultTestConfig(), Options{TagName: "yaml"})
+	require.NoError(t, err)
+	assert.Equal(t, ":8443", cfg.Server.Address, "The file extension should select the decoder")
+	assert.True(t, cfg.Server.Enabled, "TOML booleans should be decoded")
+}
+
+func TestLoadAllowsMissingFile(t *testing.T) {
+	testCases := []struct {
+		name string
+		path string
+	}{
+		{name: "empty path"},
+		{name: "unknown extension", path: filepath.Join(t.TempDir(), "missing.conf")},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			t.Setenv("APP_SERVER_ADDRESS", ":9443")
+
+			cfg, err := Load(testCase.path, defaultTestConfig(), Options{
+				TagName:      "yaml",
+				AllowMissing: true,
+			})
+			require.NoError(t, err)
+			assert.Equal(t, ":9443", cfg.Server.Address, "Environment should override defaults without a file")
+		})
+	}
+}
+
+func TestLoadSupportsRecursiveConfigTypes(t *testing.T) {
+	configPath := writeConfigFile(t, "config.yaml", `
+value: first
+next:
+  value: second
+`)
+
+	cfg, err := Load(configPath, &recursiveConfig{}, Options{
+		TagName: "yaml",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "first", cfg.Value, "Root values should be decoded")
+	require.NotNil(t, cfg.Next, "Recursive configuration should be decoded from the file")
+	assert.Equal(t, "second", cfg.Next.Value, "Nested recursive values should be decoded")
+}
+
+func TestLoadRejectsMissingFileByDefault(t *testing.T) {
+	_, err := Load(filepath.Join(t.TempDir(), "missing.yaml"), defaultTestConfig(), Options{
+		TagName: "yaml",
+	})
+	require.Error(t, err)
+}
+
+func TestLoadStrictRejectsUnknownKeys(t *testing.T) {
+	configPath := writeConfigFile(t, "config.yaml", "unknown: true\n")
+
+	_, err := Load(configPath, defaultTestConfig(), Options{
+		TagName: "yaml",
+		Strict:  true,
+	})
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "invalid keys")
+}
+
+func TestLoadUsesTagAsDefaultFileType(t *testing.T) {
+	configPath := writeConfigFile(t, "config.conf", "server:\n  address: :8443\n")
+
+	cfg, err := Load(configPath, defaultTestConfig(), Options{TagName: "yaml"})
+	require.NoError(t, err)
+	assert.Equal(t, ":8443", cfg.Server.Address, "The struct tag should select the fallback decoder")
+}
+
+func TestLoadRejectsNilDefault(t *testing.T) {
+	configPath := writeConfigFile(t, "config.yaml", "server: {}\n")
+
+	_, err := Load(configPath, (*testConfig)(nil), Options{TagName: "yaml"})
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "default config is nil")
+}
+
+func writeConfigFile(t *testing.T, name, contents string) string {
+	t.Helper()
+
+	configPath := filepath.Join(t.TempDir(), name)
+	require.NoError(t, os.WriteFile(configPath, []byte(contents), 0o600))
+	return configPath
+}

util/config/template.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+	"text/template"
+)
+
+// ExpandEnvTemplate substitutes Go-template references with environment values.
+func ExpandEnvTemplate(data []byte) ([]byte, error) {
+	tmpl, err := template.New("config").Parse(string(data))
+	if err != nil {
+		return nil, fmt.Errorf("parse environment template: %w", err)
+	}
+
+	var output bytes.Buffer
+	if err := tmpl.Execute(&output, environmentMap()); err != nil {
+		return nil, fmt.Errorf("execute environment template: %w", err)
+	}
+	return output.Bytes(), nil
+}
+
+func environmentMap() map[string]string {
+	environment := make(map[string]string)
+	for _, entry := range os.Environ() {
+		key, value, ok := strings.Cut(entry, "=")
+		if ok {
+			environment[key] = value
+		}
+	}
+	return environment
+}

util/file.go

@@ -1,7 +1,6 @@
 package util
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -10,10 +9,10 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
-	"text/template"
 
 	log "github.com/sirupsen/logrus"
+
+	configloader "github.com/netbirdio/netbird/util/config"
 )
 
 func WriteBytesWithRestrictedPermission(ctx context.Context, file string, bs []byte) error {
@@ -233,8 +232,6 @@ func ListFiles(dir, pattern string) ([]string, error) {
 
 // ReadJsonWithEnvSub reads JSON config file and maps to a provided interface with environment variable substitution
 func ReadJsonWithEnvSub(file string, res interface{}) (interface{}, error) {
-	envVars := getEnvMap()
-
 	f, err := os.Open(file)
 	if err != nil {
 		return nil, err
@@ -246,19 +243,12 @@ func ReadJsonWithEnvSub(file string, res interface{}) (interface{}, error) {
 		return nil, err
 	}
 
-	t, err := template.New("").Parse(string(bs))
+	output, err := configloader.ExpandEnvTemplate(bs)
 	if err != nil {
-		return nil, fmt.Errorf("error parsing template: %v", err)
+		return nil, err
 	}
 
-	var output bytes.Buffer
-	// Execute the template, substituting environment variables
-	err = t.Execute(&output, envVars)
-	if err != nil {
-		return nil, fmt.Errorf("error executing template: %v", err)
-	}
-
-	err = json.Unmarshal(output.Bytes(), &res)
+	err = json.Unmarshal(output, &res)
 	if err != nil {
 		return nil, fmt.Errorf("failed parsing Json file after template was executed, err: %v", err)
 	}
@@ -266,20 +256,6 @@ func ReadJsonWithEnvSub(file string, res interface{}) (interface{}, error) {
 	return res, nil
 }
 
-// getEnvMap Convert the output of os.Environ() to a map
-func getEnvMap() map[string]string {
-	envMap := make(map[string]string)
-
-	for _, env := range os.Environ() {
-		parts := strings.SplitN(env, "=", 2)
-		if len(parts) == 2 {
-			envMap[parts[0]] = parts[1]
-		}
-	}
-
-	return envMap
-}
-
 // CopyFileContents copies contents of the given src file to the dst file
 func CopyFileContents(src, dst string) (err error) {
 	in, err := os.Open(src)