[management,signal] Make the Let's Encrypt challenge listener address configurable (#7706)

* [management,signal] Make the Let's Encrypt challenge listener address configurable

With Let's Encrypt enabled and --port set to something other than 443,
signal and management also opened a separate challenge listener that was
hard-coded to :443. Non-root deployments, such as the UBI images, could
not start that listener.

Add --letsencrypt-listen-address to both. It defaults to :443, so current
behavior is unchanged. An empty value disables the separate listener for
setups that forward public port 443 to --port, where the main TLS
listener already answers TLS-ALPN-01 challenges.

Signal now fails on startup when the challenge listener cannot bind, and
exits non-zero when a server stops unexpectedly instead of exiting 0. A
failure reported before the run loop waited was previously dropped.
Management no longer opens a new :443 listener on shutdown just to close it.

* [management,signal] Keep the challenge listener change additive

Remove the Signal fail-fast changes from this PR. They change the
behavior that existing installations see after an upgrade, so they move
to a separate PR.

If the challenge listener cannot bind, Signal now logs the error and
continues. The main TLS listener still answers TLS-ALPN-01 challenges.
Management keeps its previous behavior and stops with an error.

The check for an empty address moves to the caller, so the function
does not return a nil listener with a nil error. Also add assertion
messages, guard a nil listener in a test cleanup, and add the flag to
the Signal README.

signal/cmd/run.go

@@ -36,7 +36,12 @@ import (
 	"google.golang.org/grpc/keepalive"
 )
 
-const legacyGRPCPort = 10000
+const (
+	legacyGRPCPort = 10000
+	// defaultLetsencryptListenAddress is where Let's Encrypt connects for
+	// TLS-ALPN-01 challenges unless public port 443 is forwarded elsewhere.
+	defaultLetsencryptListenAddress = ":443"
+)
 
 var (
 	signalPort               int
@@ -44,6 +49,7 @@ var (
 	signalLetsencryptDomain  string
 	signalLetsencryptEmail   string
 	signalLetsencryptDataDir string
+	signalLetsencryptListen  string
 	signalCertFile           string
 	signalCertKey            string
 
@@ -124,8 +130,18 @@ var (
 
 			grpcRootHandler := grpcHandlerFunc(grpcServer, metricsServer.Meter)
 
-			if certManager != nil {
-				startServerWithCertManager(certManager, grpcRootHandler)
+			var certListener net.Listener
+			switch {
+			case certManager == nil:
+			case signalPort != 443 && signalLetsencryptListen == "":
+				// The main TLS listener uses the cert manager's TLS config, so it still
+				// answers TLS-ALPN-01 challenges when public port 443 is forwarded to it.
+				log.Infof("LetsEncrypt challenge server disabled, challenges are answered on port %d", signalPort)
+			default:
+				certListener, err = startServerWithCertManager(certManager, grpcRootHandler)
+				if err != nil {
+					log.Errorf("LetsEncrypt challenge server not started: %v", err)
+				}
 			}
 
 			var compatListener net.Listener
@@ -169,6 +185,10 @@ var (
 			SetupCloseHandler()
 
 			<-stopCh
+			if certListener != nil {
+				_ = certListener.Close()
+				log.Infof("stopped LetsEncrypt challenge server")
+			}
 			if grpcListener != nil {
 				_ = grpcListener.Close()
 				log.Infof("stopped gRPC server")
@@ -245,18 +265,24 @@ func getTLSConfigurations() ([]grpc.ServerOption, *autocert.Manager, *tls.Config
 	return []grpc.ServerOption{grpc.Creds(transportCredentials)}, certManager, tlsConfig, err
 }
 
-func startServerWithCertManager(certManager *autocert.Manager, grpcRootHandler http.Handler) {
-	// a call to certManager.Listener() always creates a new listener so we do it once
-	httpListener := certManager.Listener()
+func startServerWithCertManager(certManager *autocert.Manager, grpcRootHandler http.Handler) (net.Listener, error) {
 	if signalPort == 443 {
+		// a call to certManager.Listener() always creates a new listener so we do it once
+		httpListener := certManager.Listener()
 		// running gRPC and HTTP cert manager on the same port
 		serveHTTP(httpListener, certManager.HTTPHandler(grpcRootHandler))
 		log.Infof("running HTTP server (LetsEncrypt challenge handler) and gRPC server on the same port: %s", httpListener.Addr().String())
-	} else {
-		// Start the HTTP cert manager server separately
-		serveHTTP(httpListener, certManager.HTTPHandler(nil))
-		log.Infof("running HTTP server (LetsEncrypt challenge handler): %s", httpListener.Addr().String())
+		return httpListener, nil
 	}
+
+	httpListener, err := tls.Listen("tcp", signalLetsencryptListen, certManager.TLSConfig())
+	if err != nil {
+		return nil, fmt.Errorf("create LetsEncrypt challenge listener on %s: %w", signalLetsencryptListen, err)
+	}
+	// Start the HTTP cert manager server separately
+	serveHTTP(httpListener, certManager.HTTPHandler(nil))
+	log.Infof("running HTTP server (LetsEncrypt challenge handler): %s", httpListener.Addr().String())
+	return httpListener, nil
 }
 
 func grpcHandlerFunc(grpcServer *grpc.Server, meter metric.Meter) http.Handler {
@@ -334,6 +360,7 @@ func init() {
 	runCmd.PersistentFlags().StringVar(&signalLetsencryptDataDir, "letsencrypt-data-dir", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.")
 	runCmd.PersistentFlags().StringVar(&signalLetsencryptDataDir, "ssl-dir", "", "server ssl directory location. *Required only for Let's Encrypt certificates. Deprecated: use --letsencrypt-data-dir")
 	runCmd.PersistentFlags().StringVar(&signalLetsencryptDomain, "letsencrypt-domain", "", "a domain to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS")
+	runCmd.PersistentFlags().StringVar(&signalLetsencryptListen, "letsencrypt-listen-address", defaultLetsencryptListenAddress, "address of the separate Let's Encrypt challenge listener, used when --port is not 443. Set it empty when public port 443 is forwarded to --port, which answers the challenges itself")
 	runCmd.PersistentFlags().StringVar(&signalLetsencryptEmail, "letsencrypt-email", "", "email address to use for Let's Encrypt certificate registration")
 	runCmd.PersistentFlags().StringVar(&signalCertFile, "cert-file", "", "Location of your SSL certificate. Can be used when you have an existing certificate and don't want a new certificate be generated automatically. If letsencrypt-domain is specified this property has no effect")
 	runCmd.PersistentFlags().StringVar(&signalCertKey, "cert-key", "", "Location of your SSL certificate private key. Can be used when you have an existing certificate and don't want a new certificate be generated automatically. If letsencrypt-domain is specified this property has no effect")

signal/cmd/run_test.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/acme"
+	"golang.org/x/crypto/acme/autocert"
+)
+
+func setLetsencryptListen(t *testing.T, port int, address string) {
+	t.Helper()
+	oldPort, oldAddress := signalPort, signalLetsencryptListen
+	signalPort, signalLetsencryptListen = port, address
+	t.Cleanup(func() {
+		signalPort, signalLetsencryptListen = oldPort, oldAddress
+	})
+}
+
+func TestStartServerWithCertManager_CustomAddress(t *testing.T) {
+	setLetsencryptListen(t, 10000, "127.0.0.1:0")
+
+	listener, err := startServerWithCertManager(&autocert.Manager{}, http.NotFoundHandler())
+	require.NoError(t, err)
+	require.NotNil(t, listener, "challenge listener should be created on the configured address")
+	t.Cleanup(func() { _ = listener.Close() })
+
+	conn, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second)
+	require.NoError(t, err)
+	require.NoError(t, conn.Close())
+}
+
+func TestStartServerWithCertManager_BindFailure(t *testing.T) {
+	occupied, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = occupied.Close() })
+	setLetsencryptListen(t, 10000, occupied.Addr().String())
+
+	listener, err := startServerWithCertManager(&autocert.Manager{}, http.NotFoundHandler())
+	require.Error(t, err)
+	require.Nil(t, listener, "no listener should be returned when the bind fails")
+}
+
+func TestCertManagerTLSConfigAnswersTLSALPN01(t *testing.T) {
+	// Disabling the separate listener relies on the main listener answering
+	// TLS-ALPN-01 challenges through the cert manager's TLS config.
+	cfg := (&autocert.Manager{}).TLSConfig()
+	require.Contains(t, cfg.NextProtos, acme.ALPNProto, "cert manager TLS config should offer the ACME TLS-ALPN protocol")
+}