diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 85ed5cd3b..a1336cdab 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -14,5 +14,15 @@ reviews: - "!**/*.ts" - "!**/*.js" - "!**/*.svg" + pre_merge_checks: + custom_checks: + - name: "No attribution trailers" + mode: error + instructions: >- + Fail when the PR description or any commit message carries an + attribution trailer or footer: Co-Authored-By, Claude-Session, + Generated-By, or a "Generated with"/"Generated by" tool line. + Contributors own their contributions (AGENTS.md); ask for the + lines to be removed. chat: auto_reply: true diff --git a/.githooks/commit-msg b/.githooks/commit-msg new file mode 100755 index 000000000..e5699bd9c --- /dev/null +++ b/.githooks/commit-msg @@ -0,0 +1,26 @@ +#!/bin/bash +# Refuses commit messages that carry attribution trailers. Contributors own +# their contributions (AGENTS.md, "No Co-Authored-By or tool-attribution +# trailers"); a trailer spreads that ownership onto a tool or a bystander. + +msg_file="$1" + +# Trailer keys in any casing, with any bullet or emoji in front. +trailers='^[^[:alnum:]]*(co-authored-by|claude-session|generated-by):' +# "Generated with/by" footers, including "Generated with by". +footer='^[^[:alnum:]]*generated (with|by)( [^[:alnum:]]*by)? ' +# A footer names a product, so a capitalized word must follow the phrase +# itself. Prose such as "generated by the protobuf compiler" stays legal. +tool='[Gg][Ee][Nn][Ee][Rr][Aa][Tt][Ee][Dd] ([Ww][Ii][Tt][Hh]|[Bb][Yy])( [^[:alnum:]]*[Bb][Yy])? [^[:alnum:]]*[A-Z]' + +offending=$( { + grep -Ein "$trailers" "$msg_file" + grep -Ein "$footer" "$msg_file" | grep -E "$tool" +} | sort -un ) + +if [ -n "$offending" ]; then + echo "commit-msg: attribution trailers are not accepted in this repository:" >&2 + printf '%s\n' "$offending" | sed 's/^/ /' >&2 + echo "Remove them and commit again (see AGENTS.md)." >&2 + exit 1 +fi diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 9e3caa17a..449eb14fa 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -514,14 +514,32 @@ jobs: if: matrix.store == 'mysql' run: docker pull mlsmaycon/warmed-mysql:8 + # The -json stream goes through tools/gotestsummary so the log shows one + # line per test, the output of failed tests, the head of a timeout panic + # with the still-running tests, and the slowest tests per package. - name: Test + shell: bash run: | + set -o pipefail CGO_ENABLED=1 GOARCH=${{ matrix.arch }} \ NETBIRD_STORE_ENGINE=${{ matrix.store }} \ CI=true \ - go test -tags=devcert -coverprofile=coverage.txt \ + go test -json -tags=devcert -coverprofile=coverage.txt \ -exec "sudo --preserve-env=CI,NETBIRD_STORE_ENGINE" \ - -timeout 20m ./management/... ./shared/management/... + -timeout 20m ./management/... ./shared/management/... \ + | tee management-test-events.jsonl \ + | go run ./tools/gotestsummary + + # The summary trims long outputs; the raw stream keeps every line for + # the failures that need it. A green run has no use for it. + - name: Upload raw test events + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: management-unit-test-events-${{ matrix.store }} + path: management-test-events.jsonl + if-no-files-found: ignore + retention-days: 14 - name: Upload coverage reports to Codecov if: matrix.arch == 'amd64' @@ -771,12 +789,27 @@ jobs: - name: check git status run: git --no-pager diff --exit-code + # Same summary as the unit job: a timeout here names the tests still + # running instead of ending in a goroutine dump. - name: Test + shell: bash run: | + set -o pipefail CGO_ENABLED=1 GOARCH=${{ matrix.arch }} \ NETBIRD_STORE_ENGINE=${{ matrix.store }} \ CI=true \ - mage integrationtest:all -gotestflags="-coverprofile=coverage.txt" + mage integrationtest:all -gotestflags="-json -coverprofile=coverage.txt" \ + | tee management-integration-test-events.jsonl \ + | go run ./tools/gotestsummary + + - name: Upload raw test events + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: management-integration-test-events-${{ matrix.store }} + path: management-integration-test-events.jsonl + if-no-files-found: ignore + retention-days: 14 - name: Upload coverage reports to Codecov if: matrix.arch == 'amd64' diff --git a/AGENTS.md b/AGENTS.md index 5497acb15..3838a7913 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,7 @@ make lint # golangci-lint on files changed vs origin/main (also the p make lint-all # full-repository lint, matches CI make test-unit # host-safe unit tests, -tags devcert, no sudo make test-privileged # privileged-tagged suite in a Docker container with NET_ADMIN -make setup-hooks # wire make lint into .githooks/pre-push +make setup-hooks # wire .githooks: pre-push runs make lint, commit-msg refuses attribution trailers # Narrow runs go test ./client/internal/dns/... diff --git a/CLAUDE.md b/CLAUDE.md index 764f406be..72681748e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1 +1,4 @@ -See [AGENTS.md](AGENTS.md) for the agent guidelines in this repository. +The agent guidelines live in [AGENTS.md](AGENTS.md). It is imported here so +every session loads it in full rather than following a pointer. + +@AGENTS.md diff --git a/Makefile b/Makefile index 0a4fad2f2..26c5b932e 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,8 @@ lint-install: $(GOLANGCI_LINT) # Setup git hooks for all developers setup-hooks: @git config core.hooksPath .githooks - @chmod +x .githooks/pre-push - @echo "✅ Git hooks configured! Pre-push will now run 'make lint'" + @chmod +x .githooks/pre-push .githooks/commit-msg + @echo "✅ Git hooks configured! Pre-push runs 'make lint'; commit-msg refuses attribution trailers" # Host-safe unit tests: excludes the privileged-tagged tests (root / system-mutating). # Runs as a normal user with no sudo and leaves host networking untouched. diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index bf36b944b..d753ee43e 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -819,8 +819,8 @@ func (d *Status) SetSessionExpiresAt(deadline time.Time) { // "none" would blank the UI at the exact moment it should say the session // ended. func (d *Status) GetSessionExpiresAt() time.Time { - d.mux.Lock() - defer d.mux.Unlock() + d.mux.RLock() + defer d.mux.RUnlock() return d.sessionExpiresAt } diff --git a/e2e/agentnetwork/agent_config_test.go b/e2e/agentnetwork/agent_config_test.go index 58bfddab3..89e7202e0 100644 --- a/e2e/agentnetwork/agent_config_test.go +++ b/e2e/agentnetwork/agent_config_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/netbirdio/netbird/e2e/harness" "github.com/netbirdio/netbird/shared/management/http/api" ) @@ -65,16 +66,24 @@ func configProvider(cfg api.AgentNetworkAgentConfig, name string) *api.AgentNetw func TestAgentConfigAllowlistOfDeclaredModels(t *testing.T) { ctx := context.Background() + // Saving a provider makes management verify the credential against the + // upstream, and a real vendor refuses the dummy key and the save with it. + // The providers point at the mock upstream instead: it resolves to a + // private address, which the check declines to dial and treats as + // unverifiable rather than as a failure, so the save goes through. The + // test is about the allowlist, not the upstream. + vllm, err := harness.StartVLLM(ctx, srv) + require.NoError(t, err, "start mock upstream") + t.Cleanup(func() { _ = vllm.Terminate(context.Background()) }) + cases := []struct { name string catalogID string - upstream string declared string }{ { name: "plain-declared-id", catalogID: "openai_api", - upstream: "https://api.openai.com", declared: "gpt-4o-mini", }, { @@ -83,7 +92,6 @@ func TestAgentConfigAllowlistOfDeclaredModels(t *testing.T) { // picker copies it as-is. name: "bedrock-declared-id", catalogID: "bedrock_api", - upstream: "https://bedrock-runtime.eu-central-1.amazonaws.com", declared: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", }, } @@ -101,7 +109,7 @@ func TestAgentConfigAllowlistOfDeclaredModels(t *testing.T) { prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ Name: providerName, ProviderId: tc.catalogID, - UpstreamUrl: tc.upstream, + UpstreamUrl: vllm.URL, ApiKey: ptr("sk-dummy-e2e-key"), Enabled: ptr(true), Models: &[]api.AgentNetworkProviderModel{{Id: tc.declared, InputPer1k: 0.001, OutputPer1k: 0.002}}, diff --git a/e2e/agentnetwork/settings_cluster_validation_test.go b/e2e/agentnetwork/settings_cluster_validation_test.go new file mode 100644 index 000000000..82c84dc74 --- /dev/null +++ b/e2e/agentnetwork/settings_cluster_validation_test.go @@ -0,0 +1,178 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// TestSettingsBootstrapValidatesProxyCluster covers the bootstrap-time check +// on the picked cluster, end to end against a real proxy. +// +// The synthesised gateway service is always private: agents reach it over the +// WireGuard tunnel and are authorised by their peer identity. Only a cluster +// with private capabilities can serve that, and management reports it per +// cluster as the `private` capability — the same supports_private flag the +// dashboard reads to decide which clusters it may offer. The endpoint assigned at bootstrap is immutable, so pinning +// to a cluster that cannot serve it has to be refused up front rather than +// leaving the account with a dead gateway. +// +// One combined server and one cluster address, walked through three states: +// a live centralised proxy (refused), that proxy stopped so nothing in the +// cluster is live any more (still refused — the record of what the cluster is +// outlives its heartbeats), and finally a private-capable proxy (accepted, the +// capability being any-true across the cluster's live proxies). Same account, +// same address, so nothing but the cluster's state accounts for the different +// answers. +func TestSettingsBootstrapValidatesProxyCluster(t *testing.T) { + ctx := context.Background() + + fresh, err := harnessStartFresh(ctx, t) + require.NoError(t, err, "start dedicated combined server") + + proxyToken, err := fresh.CreateProxyTokenCLI(ctx, "e2e-cluster-validation") + require.NoError(t, err, "mint proxy token via CLI") + + const cluster = harness.AgentNetworkCluster + + // A centralised proxy: connected and serving the cluster, but without + // private capabilities, so it cannot serve a private service. + central, err := harness.StartProxy(ctx, fresh, proxyToken, map[string]string{ + "NB_PROXY_PRIVATE": "false", + }) + require.NoError(t, err, "start centralised proxy") + // Terminated mid-test; the cleanup only covers an early failure. + t.Cleanup(func() { _ = central.Terminate(context.Background()) }) + + waitClusterPrivate(ctx, t, fresh, cluster, false) + + _, err = fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + ProxyAddress: ptr(cluster), + }) + require.Error(t, err, "bootstrap onto a cluster without private capabilities must be refused") + requireClientError(t, err) + assert.Contains(t, err.Error(), "private capabilities", + "the refusal must name what the cluster is missing: %v", err) + + after, err := fresh.GetSettings(ctx) + require.NoError(t, err, "settings must still read after a refused bootstrap") + assert.Empty(t, after.Endpoint, "a refused bootstrap must not assign an endpoint") + assert.Empty(t, after.ProxyAddress, "a refused bootstrap must not pin a cluster") + + // Stopping the centralised proxy must not turn the refusal into an + // acceptance: the cluster's proxy rows outlive their heartbeats (only the + // hourly stale reaper removes them), so the cluster is still on record as + // one that cannot serve the gateway. Judging on liveness instead would + // make "wait for the proxy to go quiet" a way to pin the account's + // immutable endpoint to a cluster that can never serve it. + require.NoError(t, central.Terminate(ctx), "stop the centralised proxy") + waitClusterAbsent(ctx, t, fresh, cluster) + + _, err = fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + ProxyAddress: ptr(cluster), + }) + require.Error(t, err, "an offline cluster without private capabilities on record must stay refused") + requireClientError(t, err) + + // Add a private-capable proxy to the same cluster: now it can serve a private + // service, and the very same request must go through. + privateProxy, err := harness.StartProxy(ctx, fresh, proxyToken) + require.NoError(t, err, "start private-capable proxy") + t.Cleanup(func() { _ = privateProxy.Terminate(context.Background()) }) + + waitClusterPrivate(ctx, t, fresh, cluster, true) + + bootstrapped, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + ProxyAddress: ptr(cluster), + }) + require.NoError(t, err, "bootstrap onto a private-capable cluster must succeed") + assert.Equal(t, cluster, bootstrapped.ProxyAddress, "the pinned cluster is the requested one") + assert.True(t, strings.HasSuffix(bootstrapped.Endpoint, "."+cluster), + "the endpoint must hang one label beneath the cluster: %s", bootstrapped.Endpoint) +} + +// waitClusterPrivate polls the domains endpoint — the list the dashboard picks +// its bootstrap cluster from — until the free domain for clusterAddr reports +// supports_private == want. A proxy's capabilities land when it registers, so +// this is the barrier between starting a proxy and asserting on what +// management thinks its cluster can do. +func waitClusterPrivate(ctx context.Context, t *testing.T, c *harness.Combined, clusterAddr string, want bool) { + t.Helper() + + deadline := time.Now().Add(90 * time.Second) + var last string + for time.Now().Before(deadline) { + domains, err := c.API().ReverseProxyDomains.List(ctx) + if err != nil { + last = "list domains: " + err.Error() + } else { + last = "cluster not listed" + for _, d := range domains { + if d.Domain != clusterAddr { + continue + } + if d.SupportsPrivate == nil { + last = "supports_private not reported yet" + break + } + if *d.SupportsPrivate == want { + return + } + last = "supports_private is not the expected value" + break + } + } + if !waitBeforeRetry(ctx, 2*time.Second) { + break + } + } + t.Fatalf("cluster %s never reported supports_private=%v: %s", clusterAddr, want, last) +} + +// waitClusterAbsent polls the domains endpoint until clusterAddr is no longer +// offered, i.e. management sees no live proxy in it. The free-domain list is +// built from the active clusters, so this is how a proxy going away becomes +// observable — while the cluster's rows, and so its capability record, remain. +// +// The budget has to clear the active window, not just the disconnect: a proxy +// that closes its stream cleanly is marked disconnected at once, but one that +// dies without that is only dropped when its last heartbeat ages past +// proxyActiveThreshold (2 minutes), so a 90s deadline could fail the test on +// the slow path alone. +func waitClusterAbsent(ctx context.Context, t *testing.T, c *harness.Combined, clusterAddr string) { + t.Helper() + + deadline := time.Now().Add(3 * time.Minute) + var last string + for time.Now().Before(deadline) { + domains, err := c.API().ReverseProxyDomains.List(ctx) + if err != nil { + last = "list domains: " + err.Error() + } else { + listed := false + for _, d := range domains { + if d.Domain == clusterAddr { + listed = true + break + } + } + if !listed { + return + } + last = "cluster still listed as active" + } + if !waitBeforeRetry(ctx, 2*time.Second) { + break + } + } + t.Fatalf("cluster %s never dropped out of the active list: %s", clusterAddr, last) +} diff --git a/management/internals/modules/agentnetwork/handlers/handlers_test.go b/management/internals/modules/agentnetwork/handlers/handlers_test.go index 6d1be3562..2b2f3b8a3 100644 --- a/management/internals/modules/agentnetwork/handlers/handlers_test.go +++ b/management/internals/modules/agentnetwork/handlers/handlers_test.go @@ -9,6 +9,7 @@ import ( "runtime" "strings" "testing" + "time" "go.uber.org/mock/gomock" "github.com/gorilla/mux" @@ -17,6 +18,7 @@ import ( "github.com/netbirdio/netbird/management/internals/modules/agentnetwork" agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" + rpproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" "github.com/netbirdio/netbird/management/server/account" nbcontext "github.com/netbirdio/netbird/management/server/context" "github.com/netbirdio/netbird/management/server/permissions" @@ -29,6 +31,9 @@ import ( const ( testAccountID = "acc-1" testUserID = "user-bob" + // testClusterAddress is the shared proxy cluster the settings tests pin + // their gateway to; the fixture seeds a connected private-capable proxy for it. + testClusterAddress = "eu.proxy.netbird.io" ) // agentNetworkHandlerFixture builds a real agentnetwork.Manager with @@ -75,6 +80,12 @@ func newAgentNetworkHandlerFixture(t *testing.T) *agentNetworkHandlerFixture { manager := agentnetwork.NewManager(st, perms, accounts, nil) h := &handler{manager: manager} + // The labeled bootstrap validates its proxy_address against the live + // clusters, so seed the shared cluster these tests pin to as a real, + // private-capable one — the wire-shape assertions then run through the + // validated path rather than the "nothing connected yet" carve-out. + seedSharedPrivateCluster(t, st, testClusterAddress) + router := mux.NewRouter() router.HandleFunc("/agent-network/providers", h.createProvider).Methods("POST") router.HandleFunc("/agent-network/providers/{providerId}", h.getProvider).Methods("GET") @@ -268,3 +279,21 @@ func TestConsumptionHandler_PopulatedAccountListsRows(t *testing.T) { assert.Equal(t, groupRow.WindowStartUtc, userRow.WindowStartUtc, "rows recorded in the same window must share the aligned window_start_utc") } + +// seedSharedPrivateCluster registers a connected, NetBird-operated proxy +// with private capabilities (the `private` capability) so +// clusterAddr is a cluster any account may pin its agent-network gateway to. +func seedSharedPrivateCluster(t *testing.T, st store.Store, clusterAddr string) { + t.Helper() + private := true + now := time.Now().UTC() + require.NoError(t, st.SaveProxy(context.Background(), &rpproxy.Proxy{ + ID: "shared-proxy-" + clusterAddr, + SessionID: "shared-session", + ClusterAddress: clusterAddr, + LastSeen: now, + ConnectedAt: &now, + Status: rpproxy.StatusConnected, + Capabilities: rpproxy.Capabilities{Private: &private}, + }), "seeding the shared proxy cluster must succeed") +} diff --git a/management/internals/modules/agentnetwork/manager.go b/management/internals/modules/agentnetwork/manager.go index efcc944be..d1a5ebd7b 100644 --- a/management/internals/modules/agentnetwork/manager.go +++ b/management/internals/modules/agentnetwork/manager.go @@ -1036,6 +1036,18 @@ func (m *managerImpl) bootstrapSelfAddressed(ctx context.Context, settings *type if err != nil { return status.Errorf(status.InvalidArgument, "invalid endpoint: %s", err) } + if err := m.requireHostNotForeign(ctx, settings.AccountID, hostname); err != nil { + return err + } + // Another account's labeled pin beneath this hostname makes it their + // cluster: a proxy serving them there would never serve this endpoint. + // The domain unique index already arbitrates two endpoints on one name. + if err := m.requireNotClaimedByOtherAccount(ctx, settings.AccountID, hostname, m.store.HasGatewayClusterPinnedByOtherAccount); err != nil { + return err + } + if err := m.validateGatewayCluster(ctx, settings.AccountID, hostname); err != nil { + return err + } settings.Domain = hostname settings.ProxyAddress = hostname @@ -1054,6 +1066,99 @@ func (m *managerImpl) bootstrapSelfAddressed(ctx context.Context, settings *type return nil } +// validateGatewayCluster rejects a bootstrap pinned to a cluster that cannot +// serve the account's gateway — a labeled endpoint beneath the cluster and a +// self-addressed one on the very address a proxy declares alike, since the +// service behind either is the same private one. +// +// The synthesised gateway service is unconditionally private +// (buildAccountService): agents reach it over the WireGuard tunnel and are +// authorised by ValidateTunnelPeer against the policies' source groups, and +// its single target is the cluster itself with DirectUpstream. Only a cluster +// with private capabilities can serve that. Management reports it per cluster +// as the `private` capability, the same flag the dashboard renders as +// supports_private when it gates NetBird-only services. +// +// Without this check the bootstrap happily pins to any cluster the caller +// names, including one without private capabilities — and the endpoint it +// allocates is immutable, so the account is left with a dead gateway that only +// a DeleteSettings/re-bootstrap can undo. +// +// Whether management knows the cluster is decided on the proxy rows +// themselves, never on how fresh their heartbeats are: a cluster's rows +// outlive its proxies' liveness (only the stale-proxy reaper removes them), so +// a cluster that exists stays judged as one. Judging on liveness instead would +// make the same centralised cluster pass or fail depending on whether its +// proxies happened to have heartbeated in the last couple of minutes. +// +// The single opening left is a cluster management holds no proxy row for at +// all: pinning ahead of a proxy's first connection is a legitimate order — the +// dedicated path claims an address the same way, before any proxy declares it. +func (m *managerImpl) validateGatewayCluster(ctx context.Context, accountID, clusterAddr string) error { + declared, err := m.accountClusterSpellings(ctx, accountID, clusterAddr) + if err != nil { + return err + } + if len(declared) == 0 { + // No proxy has ever declared this address: an address-first pin. + return nil + } + + // A cluster management knows has to prove it can serve the gateway, and + // only a live proxy reporting the capability proves that. Both an explicit false and an + // unreported capability (nothing live in the cluster, or proxies predating + // capability reporting) fail here: unusable and unproven are the same + // answer for a decision that cannot be revisited later. + // + // The capability is read per declared spelling and taken as any-true, the + // same way it aggregates over a cluster's proxies: the store matches + // cluster_address exactly, so a host two proxies spelled differently must + // not come back unproven just because it was asked about under one of them. + for _, address := range declared { + if private := m.store.GetClusterSupportsPrivate(ctx, address); private != nil && *private { + return nil + } + } + + return status.Errorf(status.InvalidArgument, + "proxy cluster %s has no private capabilities: the agent network gateway requires a reverse proxy cluster "+ + "with private capabilities", clusterAddr) +} + +// accountClusterSpellings returns every proxy cluster address in the account's +// view — its own (BYOP) clusters plus the shared ones — that names the same +// host as clusterAddr. Empty means management holds no proxy row for that host +// in this account's view. +// +// A proxy declares its cluster address as the operator spelled it, so identity +// is compared on the normalised form rather than byte-equal — an in-memory pass +// over the account's clusters, not a query. What comes back is the stored +// spelling, because the capability lookup matches cluster_address exactly and +// would silently find nothing under a spelling the store never held. The +// cluster listing is not gated on heartbeats, so this answer does not change +// while a cluster's proxies are merely offline. +func (m *managerImpl) accountClusterSpellings(ctx context.Context, accountID, clusterAddr string) ([]string, error) { + clusters, err := m.store.GetProxyClusters(ctx, accountID) + if err != nil { + return nil, fmt.Errorf("list proxy clusters: %w", err) + } + + var spellings []string + for _, cluster := range clusters { + normalized, err := types.NormalizeHostname(cluster.Address) + if err != nil { + // An address declared in a shape we cannot normalise is not one an + // endpoint can be allocated beneath. + log.WithContext(ctx).Debugf("skipping unusable proxy cluster address %q: %s", cluster.Address, err) + continue + } + if normalized == clusterAddr { + spellings = append(spellings, cluster.Address) + } + } + return spellings, nil +} + // bootstrapLabeled allocates a labeled endpoint one label beneath the given // cluster address: Domain =