netbird/signal/cmd/run.go

package cmd

import (
	"context"
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"net"
	"net/http"
	// nolint:gosec
	_ "net/http/pprof"
	"os"
	"time"

	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
	"go.opentelemetry.io/otel/metric"
	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/net/http2"
	"golang.org/x/net/http2/h2c" //nolint:staticcheck

	"github.com/netbirdio/netbird/shared/metrics"

	"github.com/netbirdio/netbird/encryption"
	"github.com/netbirdio/netbird/shared/signal/proto"
	"github.com/netbirdio/netbird/signal/server"
	"github.com/netbirdio/netbird/util"
	"github.com/netbirdio/netbird/util/wsproxy"
	wsproxyserver "github.com/netbirdio/netbird/util/wsproxy/server"
	"github.com/netbirdio/netbird/version"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/keepalive"
)

const (
	legacyGRPCPort = 10000
	// defaultLetsencryptListenAddress is where Let's Encrypt connects for
	// TLS-ALPN-01 challenges unless public port 443 is forwarded elsewhere.
	defaultLetsencryptListenAddress = ":443"
)

var (
	signalPort               int
	metricsPort              int
	signalLetsencryptDomain  string
	signalLetsencryptEmail   string
	signalLetsencryptDataDir string
	signalLetsencryptListen  string
	signalCertFile           string
	signalCertKey            string

	signalKaep = grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
		MinTime:             5 * time.Second,
		PermitWithoutStream: true,
	})

	signalKasp = grpc.KeepaliveParams(keepalive.ServerParameters{
		MaxConnectionIdle:     15 * time.Second,
		MaxConnectionAgeGrace: 5 * time.Second,
		Time:                  5 * time.Second,
		Timeout:               2 * time.Second,
	})

	runCmd = &cobra.Command{
		Use:          "run",
		Short:        "start NetBird Signal Server daemon",
		SilenceUsage: true,
		PreRunE: func(cmd *cobra.Command, args []string) error {
			err := util.InitLog(logLevel, logFile)
			if err != nil {
				return fmt.Errorf("failed initializing log: %w", err)
			}

			flag.Parse()

			// detect whether user specified a port
			userPort := cmd.Flag("port").Changed

			var tlsEnabled bool
			if signalLetsencryptDomain != "" || (signalCertFile != "" && signalCertKey != "") {
				tlsEnabled = true
			}

			if !userPort {
				// different defaults for signalPort
				if tlsEnabled {
					signalPort = 443
				} else {
					signalPort = 80
				}
			}

			return nil
		},
		RunE: func(cmd *cobra.Command, args []string) error {
			flag.Parse()

			startPprof()

			opts, certManager, tlsConfig, err := getTLSConfigurations()
			if err != nil {
				return err
			}

			metricsServer, err := metrics.NewServer(metricsPort, "")
			if err != nil {
				return fmt.Errorf("setup metrics: %v", err)
			}

			opts = append(opts, signalKaep, signalKasp, grpc.StatsHandler(otelgrpc.NewServerHandler()))
			grpcServer := grpc.NewServer(opts...)

			go func() {
				log.Infof("running metrics server: %s%s", metricsServer.Addr, metricsServer.Endpoint)
				if err := metricsServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
					log.Fatalf("Failed to start metrics server: %v", err)
				}
			}()

			srv, err := server.NewServer(cmd.Context(), metricsServer.Meter)
			if err != nil {
				return fmt.Errorf("creating signal server: %v", err)
			}
			defer srv.Stop()
			proto.RegisterSignalExchangeServer(grpcServer, srv)

			grpcRootHandler := grpcHandlerFunc(grpcServer, metricsServer.Meter)

			var certListener net.Listener
			switch {
			case certManager == nil:
			case signalPort != 443 && signalLetsencryptListen == "":
				// The main TLS listener uses the cert manager's TLS config, so it still
				// answers TLS-ALPN-01 challenges when public port 443 is forwarded to it.
				log.Infof("LetsEncrypt challenge server disabled, challenges are answered on port %d", signalPort)
			default:
				certListener, err = startServerWithCertManager(certManager, grpcRootHandler)
				if err != nil {
					log.Errorf("LetsEncrypt challenge server not started: %v", err)
				}
			}

			var compatListener net.Listener
			var grpcListener net.Listener
			var httpListener net.Listener

			// Start the main server - always serve HTTP with WebSocket proxy support
			// If certManager is configured and signalPort == 443, it's already handled by startServerWithCertManager
			if tlsConfig == nil {
				// Without TLS, serve plain HTTP
				httpListener, err = net.Listen("tcp", fmt.Sprintf(":%d", signalPort))
				if err != nil {
					return err
				}
				log.Infof("running HTTP server with WebSocket proxy (no TLS): %s", httpListener.Addr().String())
				serveHTTP(httpListener, grpcRootHandler)
			} else if certManager == nil || signalPort != 443 {
				// Serve HTTPS if not already handled by startServerWithCertManager
				// (custom certificates or Let's Encrypt with custom port)
				httpListener, err = tls.Listen("tcp", fmt.Sprintf(":%d", signalPort), tlsConfig)
				if err != nil {
					return err
				}
				log.Infof("running HTTPS server with WebSocket proxy: %s", httpListener.Addr().String())
				serveHTTP(httpListener, grpcRootHandler)
			}

			if signalPort != legacyGRPCPort {
				// The Signal gRPC server was running on port 10000 previously. Old agents that are already connected to Signal
				// are using port 10000. For compatibility purposes we keep running a 2nd gRPC server on port 10000.
				compatListener, err = serveGRPC(grpcServer, legacyGRPCPort)
				if err != nil {
					return err
				}
				log.Infof("running gRPC backward compatibility server: %s", compatListener.Addr().String())
			}

			log.Infof("signal server version %s", version.NetbirdVersion())
			log.Infof("started Signal Service")

			SetupCloseHandler()

			<-stopCh
			if certListener != nil {
				_ = certListener.Close()
				log.Infof("stopped LetsEncrypt challenge server")
			}
			if grpcListener != nil {
				_ = grpcListener.Close()
				log.Infof("stopped gRPC server")
			}
			if httpListener != nil {
				_ = httpListener.Close()
				log.Infof("stopped HTTP server")
			}
			if compatListener != nil {
				_ = compatListener.Close()
				log.Infof("stopped gRPC backward compatibility server")
			}

			ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second)
			defer cancel()
			if err := metricsServer.Shutdown(ctx); err != nil {
				log.Errorf("Failed to stop metrics server: %v", err)
			}
			log.Infof("stopped metrics server")

			log.Infof("stopped Signal Service")

			return nil
		},
	}
)

func startPprof() {
	if pprofAddr := os.Getenv("NB_PPROF_ADDR"); pprofAddr != "" {
		log.Infof("pprof enabled, listening on: %s", pprofAddr)
		go func() {
			if err := http.ListenAndServe(pprofAddr, nil); err != nil {
				log.Fatalf("pprof server failed: %v", err)
			}
		}()
	}
}

func getTLSConfigurations() ([]grpc.ServerOption, *autocert.Manager, *tls.Config, error) {
	var (
		err         error
		certManager *autocert.Manager
		tlsConfig   *tls.Config
	)

	if signalLetsencryptDomain == "" && signalCertFile == "" && signalCertKey == "" {
		log.Infof("running without TLS")
		return nil, nil, nil, nil
	}

	if signalLetsencryptDomain != "" {
		certManager, err = encryption.CreateCertManager(signalLetsencryptDataDir, signalLetsencryptDomain)
		if err != nil {
			return nil, certManager, nil, err
		}
		tlsConfig = certManager.TLSConfig()
		log.Infof("setting up TLS with LetsEncrypt.")
	} else {
		if signalCertFile == "" || signalCertKey == "" {
			log.Errorf("both cert-file and cert-key must be provided when not using LetsEncrypt")
			return nil, certManager, nil, errors.New("both cert-file and cert-key must be provided when not using LetsEncrypt")
		}

		tlsConfig, err = loadTLSConfig(signalCertFile, signalCertKey)
		if err != nil {
			log.Errorf("cannot load TLS credentials: %v", err)
			return nil, certManager, nil, err
		}
		log.Infof("setting up TLS with custom certificates.")
	}

	transportCredentials := credentials.NewTLS(tlsConfig)

	return []grpc.ServerOption{grpc.Creds(transportCredentials)}, certManager, tlsConfig, err
}

func startServerWithCertManager(certManager *autocert.Manager, grpcRootHandler http.Handler) (net.Listener, error) {
	if signalPort == 443 {
		// a call to certManager.Listener() always creates a new listener so we do it once
		httpListener := certManager.Listener()
		// running gRPC and HTTP cert manager on the same port
		serveHTTP(httpListener, certManager.HTTPHandler(grpcRootHandler))
		log.Infof("running HTTP server (LetsEncrypt challenge handler) and gRPC server on the same port: %s", httpListener.Addr().String())
		return httpListener, nil
	}

	httpListener, err := tls.Listen("tcp", signalLetsencryptListen, certManager.TLSConfig())
	if err != nil {
		return nil, fmt.Errorf("create LetsEncrypt challenge listener on %s: %w", signalLetsencryptListen, err)
	}
	// Start the HTTP cert manager server separately
	serveHTTP(httpListener, certManager.HTTPHandler(nil))
	log.Infof("running HTTP server (LetsEncrypt challenge handler): %s", httpListener.Addr().String())
	return httpListener, nil
}

func grpcHandlerFunc(grpcServer *grpc.Server, meter metric.Meter) http.Handler {
	wsProxy := wsproxyserver.New(grpcServer, wsproxyserver.WithOTelMeter(meter))

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case wsproxy.ProxyPath + wsproxy.SignalComponent:
			wsProxy.Handler().ServeHTTP(w, r)
		default:
			grpcServer.ServeHTTP(w, r)
		}
	})
}

func notifyStop(msg string) {
	select {
	case stopCh <- 1:
		log.Error(msg)
	default:
		// stop has been already called, nothing to report
	}
}

func serveHTTP(httpListener net.Listener, handler http.Handler) {
	go func() {
		// Use h2c to support HTTP/2 without TLS (needed for gRPC)
		h1s := &http.Server{
			//nolint:staticcheck // h2c also handles the HTTP/1 Upgrade mechanism, which http.Server's UnencryptedHTTP2 does not
			Handler: h2c.NewHandler(handler, &http2.Server{}),
		}
		err := h1s.Serve(httpListener)
		if err != nil {
			notifyStop(fmt.Sprintf("failed running HTTP server %v", err))
		}
	}()
}

func serveGRPC(grpcServer *grpc.Server, port int) (net.Listener, error) {
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		return nil, err
	}
	go func() {
		err := grpcServer.Serve(listener)
		if err != nil {
			notifyStop(fmt.Sprintf("failed running gRPC server on port %d: %v", port, err))
		}
	}()
	return listener, nil
}

func loadTLSConfig(certFile string, certKey string) (*tls.Config, error) {
	// Load server's certificate and private key
	serverCert, err := tls.LoadX509KeyPair(certFile, certKey)
	if err != nil {
		return nil, err
	}

	// NewDefaultAppMetrics the credentials and return it
	config := &tls.Config{
		Certificates: []tls.Certificate{serverCert},
		ClientAuth:   tls.NoClientCert,
		NextProtos: []string{
			"h2", "http/1.1", // enable HTTP/2
		},
	}

	return config, nil
}

func init() {
	runCmd.PersistentFlags().IntVar(&signalPort, "port", 80, "Server port to listen on (defaults to 443 if TLS is enabled, 80 otherwise")
	runCmd.Flags().IntVar(&metricsPort, "metrics-port", 9090, "metrics endpoint http port. Metrics are accessible under host:metrics-port/metrics")
	runCmd.PersistentFlags().StringVar(&signalLetsencryptDataDir, "letsencrypt-data-dir", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.")
	runCmd.PersistentFlags().StringVar(&signalLetsencryptDataDir, "ssl-dir", "", "server ssl directory location. *Required only for Let's Encrypt certificates. Deprecated: use --letsencrypt-data-dir")
	runCmd.PersistentFlags().StringVar(&signalLetsencryptDomain, "letsencrypt-domain", "", "a domain to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS")
	runCmd.PersistentFlags().StringVar(&signalLetsencryptListen, "letsencrypt-listen-address", defaultLetsencryptListenAddress, "address of the separate Let's Encrypt challenge listener, used when --port is not 443. Set it empty when public port 443 is forwarded to --port, which answers the challenges itself")
	runCmd.PersistentFlags().StringVar(&signalLetsencryptEmail, "letsencrypt-email", "", "email address to use for Let's Encrypt certificate registration")
	runCmd.PersistentFlags().StringVar(&signalCertFile, "cert-file", "", "Location of your SSL certificate. Can be used when you have an existing certificate and don't want a new certificate be generated automatically. If letsencrypt-domain is specified this property has no effect")
	runCmd.PersistentFlags().StringVar(&signalCertKey, "cert-key", "", "Location of your SSL certificate private key. Can be used when you have an existing certificate and don't want a new certificate be generated automatically. If letsencrypt-domain is specified this property has no effect")
	setFlagsFromEnvVars(runCmd)
}