[management] Add SetNX and GetDel cache store operations (#7084)

management/server/cache/memory.go

@@ -0,0 +1,57 @@
+package cache
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/eko/gocache/lib/v4/store"
+	gocachestore "github.com/eko/gocache/store/go_cache/v4"
+	gocache "github.com/patrickmn/go-cache"
+)
+
+type goCacheStore struct {
+	store.StoreInterface
+	client *gocache.Cache
+	mu     sync.Mutex
+}
+
+func newMemoryStore(maxTimeout, cleanupInterval time.Duration) Store {
+	client := gocache.New(maxTimeout, cleanupInterval)
+	return &goCacheStore{
+		StoreInterface: gocachestore.NewGoCache(client),
+		client:         client,
+	}
+}
+
+func (s *goCacheStore) SetNX(_ context.Context, key, value string, ttl time.Duration) (bool, error) {
+	// Add only returns an error when a non-expired entry already exists.
+	if err := s.client.Add(key, value, ttl); err != nil {
+		return false, nil //nolint:nilerr
+	}
+	return true, nil
+}
+
+// GetDel reads the value under key and removes it. go-cache has no native read-and-delete
+// and releases its own lock between the two calls, so mu holds the pair together and no
+// value is consumed twice.
+//
+// Writes do not take mu: a Set landing mid-pair is lost, since GetDel returns the prior
+// value and deletes the new one. Callers must write a consumed key only once.
+func (s *goCacheStore) GetDel(_ context.Context, key string) (string, bool, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	value, found := s.client.Get(key)
+	if !found {
+		return "", false, nil
+	}
+	s.client.Delete(key)
+
+	str, ok := value.(string)
+	if !ok {
+		return "", false, fmt.Errorf("cached value is %T, not a string", value)
+	}
+	return str, true, nil
+}

management/server/cache/memory_test.go

@@ -0,0 +1,76 @@
+package cache_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/management/server/cache"
+)
+
+func TestMemoryStore(t *testing.T) {
+	memStore, err := cache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100)
+	require.NoError(t, err, "couldn't create memory store")
+
+	ctx := context.Background()
+	key, value := "testing", "tested"
+	err = memStore.Set(ctx, key, value)
+	assert.NoError(t, err, "couldn't set testing data")
+
+	result, err := memStore.Get(ctx, key)
+	assert.NoError(t, err, "couldn't get testing data")
+	assert.Equal(t, value, result, "value returned doesn't match testing data")
+
+	created, err := memStore.SetNX(ctx, "conditional", value, 100*time.Millisecond)
+	require.NoError(t, err, "couldn't conditionally set testing data")
+	require.True(t, created, "first conditional set should create the entry")
+
+	created, err = memStore.SetNX(ctx, "conditional", value, 100*time.Millisecond)
+	require.NoError(t, err, "couldn't conditionally check testing data")
+	require.False(t, created, "second conditional set should not replace the entry")
+
+	// test expiration
+	time.Sleep(300 * time.Millisecond)
+	_, err = memStore.Get(ctx, key)
+	assert.Error(t, err, "value should not be found")
+}
+
+func TestMemoryStoreGetDel(t *testing.T) {
+	ctx := context.Background()
+	newStore := func(t *testing.T) cache.Store {
+		t.Helper()
+		memStore, err := cache.NewStore(ctx, time.Minute, time.Minute, 100)
+		require.NoError(t, err, "couldn't create memory store")
+
+		return memStore
+	}
+
+	const (
+		key   = "consume"
+		value = "verifier"
+	)
+
+	t.Run("exactly one concurrent caller consumes the key", func(t *testing.T) {
+		memStore := newStore(t)
+		require.NoError(t, memStore.Set(ctx, key, value), "couldn't set testing data")
+
+		assertGetDelConsumedOnce(ctx, t, []cache.Store{memStore}, key, value)
+		assertGetDelMisses(ctx, t, memStore, key)
+	})
+
+	t.Run("missing key is not an error", func(t *testing.T) {
+		assertGetDelMisses(ctx, t, newStore(t), "never-set")
+	})
+
+	t.Run("expired key is not found", func(t *testing.T) {
+		memStore := newStore(t)
+		_, err := memStore.SetNX(ctx, key, value, 50*time.Millisecond)
+		require.NoError(t, err, "couldn't set testing data")
+
+		time.Sleep(100 * time.Millisecond)
+		assertGetDelMisses(ctx, t, memStore, key)
+	})
+}

management/server/cache/redis.go

@@ -0,0 +1,63 @@
+package cache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/eko/gocache/lib/v4/store"
+	redisstore "github.com/eko/gocache/store/redis/v4"
+	"github.com/redis/go-redis/v9"
+	log "github.com/sirupsen/logrus"
+)
+
+type redisStore struct {
+	store.StoreInterface
+	client *redis.Client
+}
+
+func getRedisStore(ctx context.Context, redisEnvAddr string, maxConn int) (Store, error) {
+	options, err := redis.ParseURL(redisEnvAddr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing redis cache url: %s", err)
+	}
+
+	options.MaxIdleConns = int(math.Ceil(float64(maxConn) * 0.5)) // 50% of max conns
+	options.MinIdleConns = int(math.Ceil(float64(maxConn) * 0.1)) // 10% of max conns
+	options.MaxActiveConns = maxConn
+	options.ConnMaxIdleTime = 30 * time.Minute
+	options.ConnMaxLifetime = 0
+	options.PoolTimeout = 10 * time.Second
+	redisClient := redis.NewClient(options)
+	subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+	defer cancel()
+
+	_, err = redisClient.Ping(subCtx).Result()
+	if err != nil {
+		return nil, err
+	}
+
+	log.WithContext(subCtx).Infof("using redis cache at %s", redisEnvAddr)
+
+	return &redisStore{
+		StoreInterface: redisstore.NewRedis(redisClient),
+		client:         redisClient,
+	}, nil
+}
+
+func (s *redisStore) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) {
+	return s.client.SetNX(ctx, key, value, ttl).Result()
+}
+
+func (s *redisStore) GetDel(ctx context.Context, key string) (string, bool, error) {
+	value, err := s.client.GetDel(ctx, key).Result()
+	if errors.Is(err, redis.Nil) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return value, true, nil
+}

management/server/cache/redis_test.go

@@ -0,0 +1,153 @@
+package cache_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/eko/gocache/lib/v4/store"
+	"github.com/redis/go-redis/v9"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	testcontainersredis "github.com/testcontainers/testcontainers-go/modules/redis"
+
+	"github.com/netbirdio/netbird/management/server/cache"
+)
+
+func startRedis(t *testing.T) string {
+	t.Helper()
+
+	ctx := context.Background()
+	redisContainer, err := testcontainersredis.Run(ctx, "redis:7")
+	require.NoError(t, err, "couldn't start redis container")
+
+	t.Cleanup(func() {
+		if err := redisContainer.Terminate(ctx); err != nil {
+			t.Logf("failed to terminate container: %s", err)
+		}
+	})
+
+	redisURL, err := redisContainer.ConnectionString(ctx)
+	require.NoError(t, err, "couldn't get connection string")
+
+	t.Setenv(cache.RedisStoreEnvVar, redisURL)
+	return redisURL
+}
+
+func newRedisStore(t *testing.T) cache.Store {
+	t.Helper()
+
+	redisStore, err := cache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100)
+	require.NoError(t, err)
+
+	return redisStore
+}
+
+func TestRedisStoreConnectionFailure(t *testing.T) {
+	t.Setenv(cache.RedisStoreEnvVar, "redis://127.0.0.1:6379")
+	_, err := cache.NewStore(context.Background(), 10*time.Millisecond, 30*time.Millisecond, 100)
+	require.Error(t, err, "getting redis cache store should return error")
+}
+
+func TestRedisStoreConnectionSuccess(t *testing.T) {
+	ctx := context.Background()
+	redisURL := startRedis(t)
+	redisStore := newRedisStore(t)
+
+	key, value := "testing", "tested"
+	err := redisStore.Set(ctx, key, value, store.WithExpiration(100*time.Millisecond))
+	assert.NoError(t, err, "couldn't set testing data")
+
+	result, err := redisStore.Get(ctx, key)
+	assert.NoError(t, err, "couldn't get testing data")
+	assert.Equal(t, value, result, "value returned doesn't match testing data")
+
+	options, err := redis.ParseURL(redisURL)
+	require.NoError(t, err, "parsing redis cache url")
+
+	redisClient := redis.NewClient(options)
+	r, err := redisClient.Get(ctx, key).Result()
+	assert.NoError(t, err, "couldn't get testing data from redis")
+	assert.Equal(t, value, r, "value returned from redis doesn't match testing data")
+
+	// test expiration
+	time.Sleep(300 * time.Millisecond)
+	_, err = redisStore.Get(ctx, key)
+	assert.Error(t, err, "value should not be found")
+}
+
+func TestRedisStoreSetNX(t *testing.T) {
+	ctx := context.Background()
+	redisURL := startRedis(t)
+	redisStore, secondRedisStore := newRedisStore(t), newRedisStore(t)
+
+	const (
+		key   = "conditional"
+		value = "tested"
+	)
+
+	start := make(chan struct{})
+	type setResult struct {
+		created bool
+		err     error
+	}
+	results := make(chan setResult, 2)
+	for _, cacheStore := range []cache.Store{redisStore, secondRedisStore} {
+		go func() {
+			<-start
+			created, err := cacheStore.SetNX(ctx, key, value, time.Minute)
+			results <- setResult{created: created, err: err}
+		}()
+	}
+	close(start)
+
+	created := 0
+	for range 2 {
+		result := <-results
+		require.NoError(t, result.err, "conditional redis set failed")
+		if result.created {
+			created++
+		}
+	}
+	require.Equal(t, 1, created, "expected exactly one redis client to create the entry")
+
+	options, err := redis.ParseURL(redisURL)
+	require.NoError(t, err, "parsing redis cache url")
+
+	ttl, err := redis.NewClient(options).PTTL(ctx, key).Result()
+	require.NoError(t, err, "couldn't read entry TTL")
+	require.Positive(t, ttl, "created entry should have a positive TTL")
+}
+
+func TestRedisStoreGetDel(t *testing.T) {
+	ctx := context.Background()
+	startRedis(t)
+	redisStore, secondRedisStore := newRedisStore(t), newRedisStore(t)
+
+	const (
+		key   = "consume"
+		value = "verifier"
+	)
+
+	t.Run("exactly one caller across independent clients consumes the key", func(t *testing.T) {
+		// A generous TTL: the key is consumed explicitly, so expiry racing the
+		// concurrent callers would only make the test flaky on a loaded runner.
+		err := redisStore.Set(ctx, key, value, store.WithExpiration(time.Minute))
+		require.NoError(t, err, "couldn't set value to consume")
+
+		assertGetDelConsumedOnce(ctx, t, []cache.Store{redisStore, secondRedisStore}, key, value)
+		assertGetDelMisses(ctx, t, secondRedisStore, key)
+	})
+
+	t.Run("missing key is not an error", func(t *testing.T) {
+		assertGetDelMisses(ctx, t, redisStore, "never-set")
+	})
+
+	t.Run("expired key is not found", func(t *testing.T) {
+		err := redisStore.Set(ctx, key, value, store.WithExpiration(50*time.Millisecond))
+		require.NoError(t, err, "couldn't set value to consume")
+
+		time.Sleep(100 * time.Millisecond)
+		assertGetDelMisses(ctx, t, redisStore, key)
+	})
+}

management/server/cache/store.go

@@ -2,17 +2,10 @@ package cache
 
 import (
 	"context"
-	"fmt"
-	"math"
 	"os"
 	"time"
 
 	"github.com/eko/gocache/lib/v4/store"
-	gocache_store "github.com/eko/gocache/store/go_cache/v4"
-	redis_store "github.com/eko/gocache/store/redis/v4"
-	gocache "github.com/patrickmn/go-cache"
-	"github.com/redis/go-redis/v9"
-	log "github.com/sirupsen/logrus"
 )
 
 // RedisStoreEnvVar is the environment variable that determines if a redis store should be used.
@@ -31,15 +24,23 @@ const (
 	DefaultStoreMaxConn = 1000
 )
 
+// Store extends the shared cache interface with conditional and consuming operations.
+type Store interface {
+	store.StoreInterface
+	// SetNX stores a value with a TTL only when the key does not exist.
+	SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error)
+	// GetDel reads a value and removes it, so only one caller can consume a key.
+	GetDel(ctx context.Context, key string) (value string, found bool, err error)
+}
+
 // NewStore creates a new cache store with the given max timeout and cleanup interval. It checks for the environment Variable RedisStoreEnvVar
 // to determine if a redis store should be used. If the environment variable is set, it will attempt to connect to the redis store.
-func NewStore(ctx context.Context, maxTimeout, cleanupInterval time.Duration, maxConn int) (store.StoreInterface, error) {
+func NewStore(ctx context.Context, maxTimeout, cleanupInterval time.Duration, maxConn int) (Store, error) {
 	redisAddr := GetAddrFromEnv()
 	if redisAddr != "" {
 		return getRedisStore(ctx, redisAddr, maxConn)
 	}
-	goc := gocache.New(maxTimeout, cleanupInterval)
-	return gocache_store.NewGoCache(goc), nil
+	return newMemoryStore(maxTimeout, cleanupInterval), nil
 }
 
 // GetAddrFromEnv returns the redis address from the environment variable RedisStoreEnvVar or its legacy counterpart.
@@ -50,29 +51,3 @@ func GetAddrFromEnv() string {
 	}
 	return addr
 }
-
-func getRedisStore(ctx context.Context, redisEnvAddr string, maxConn int) (store.StoreInterface, error) {
-	options, err := redis.ParseURL(redisEnvAddr)
-	if err != nil {
-		return nil, fmt.Errorf("parsing redis cache url: %s", err)
-	}
-
-	options.MaxIdleConns = int(math.Ceil(float64(maxConn) * 0.5)) // 50% of max conns
-	options.MinIdleConns = int(math.Ceil(float64(maxConn) * 0.1)) // 10% of max conns
-	options.MaxActiveConns = maxConn
-	options.ConnMaxIdleTime = 30 * time.Minute
-	options.ConnMaxLifetime = 0
-	options.PoolTimeout = 10 * time.Second
-	redisClient := redis.NewClient(options)
-	subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
-	defer cancel()
-
-	_, err = redisClient.Ping(subCtx).Result()
-	if err != nil {
-		return nil, err
-	}
-
-	log.WithContext(subCtx).Infof("using redis cache at %s", redisEnvAddr)
-
-	return redis_store.NewRedis(redisClient), nil
-}

management/server/cache/store_test.go

@@ -3,101 +3,53 @@ package cache_test
 import (
 	"context"
 	"testing"
-	"time"
 
-	"github.com/eko/gocache/lib/v4/store"
-	"github.com/redis/go-redis/v9"
-	testcontainersredis "github.com/testcontainers/testcontainers-go/modules/redis"
+	"github.com/stretchr/testify/require"
 
 	"github.com/netbirdio/netbird/management/server/cache"
 )
 
-func TestMemoryStore(t *testing.T) {
-	memStore, err := cache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100)
-	if err != nil {
-		t.Fatalf("couldn't create memory store: %s", err)
-	}
-	ctx := context.Background()
-	key, value := "testing", "tested"
-	err = memStore.Set(ctx, key, value)
-	if err != nil {
-		t.Errorf("couldn't set testing data: %s", err)
-	}
-	result, err := memStore.Get(ctx, key)
-	if err != nil {
-		t.Errorf("couldn't get testing data: %s", err)
-	}
-	if value != result.(string) {
-		t.Errorf("value returned doesn't match testing data, got %s, expected %s", result, value)
-	}
-	// test expiration
-	time.Sleep(300 * time.Millisecond)
-	_, err = memStore.Get(ctx, key)
-	if err == nil {
-		t.Error("value should not be found")
-	}
-}
+func assertGetDelConsumedOnce(ctx context.Context, t *testing.T, stores []cache.Store, key, value string) {
+	t.Helper()
 
-func TestRedisStoreConnectionFailure(t *testing.T) {
-	t.Setenv(cache.RedisStoreEnvVar, "redis://127.0.0.1:6379")
-	_, err := cache.NewStore(context.Background(), 10*time.Millisecond, 30*time.Millisecond, 100)
-	if err == nil {
-		t.Fatal("getting redis cache store should return error")
-	}
-}
+	const getDelAttempts = 64
 
-func TestRedisStoreConnectionSuccess(t *testing.T) {
-	ctx := context.Background()
-	redisContainer, err := testcontainersredis.Run(ctx, "redis:7")
-	if err != nil {
-		t.Fatalf("couldn't start redis container: %s", err)
+	type getDelResult struct {
+		value string
+		found bool
+		err   error
 	}
-	defer func() {
-		if err := redisContainer.Terminate(ctx); err != nil {
-			t.Logf("failed to terminate container: %s", err)
+
+	start := make(chan struct{})
+	results := make(chan getDelResult, getDelAttempts)
+	for i := range getDelAttempts {
+		cacheStore := stores[i%len(stores)]
+		go func() {
+			<-start
+			value, found, err := cacheStore.GetDel(ctx, key)
+			results <- getDelResult{value: value, found: found, err: err}
+		}()
+	}
+	close(start)
+
+	consumers := 0
+	for range getDelAttempts {
+		result := <-results
+		require.NoError(t, result.err, "concurrent GetDel failed")
+		if !result.found {
+			continue
 		}
-	}()
-	redisURL, err := redisContainer.ConnectionString(ctx)
-	if err != nil {
-		t.Fatalf("couldn't get connection string: %s", err)
-	}
-
-	t.Setenv(cache.RedisStoreEnvVar, redisURL)
-	redisStore, err := cache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100)
-	if err != nil {
-		t.Fatalf("couldn't create redis store: %s", err)
-	}
-
-	key, value := "testing", "tested"
-	err = redisStore.Set(ctx, key, value, store.WithExpiration(100*time.Millisecond))
-	if err != nil {
-		t.Errorf("couldn't set testing data: %s", err)
-	}
-	result, err := redisStore.Get(ctx, key)
-	if err != nil {
-		t.Errorf("couldn't get testing data: %s", err)
-	}
-	if value != result.(string) {
-		t.Errorf("value returned doesn't match testing data, got %s, expected %s", result, value)
-	}
-
-	options, err := redis.ParseURL(redisURL)
-	if err != nil {
-		t.Errorf("parsing redis cache url: %s", err)
-	}
-
-	redisClient := redis.NewClient(options)
-	r, e := redisClient.Get(ctx, key).Result()
-	if e != nil {
-		t.Errorf("couldn't get testing data from redis: %s", e)
-	}
-	if value != r {
-		t.Errorf("value returned from redis doesn't match testing data, got %s, expected %s", r, value)
-	}
-	// test expiration
-	time.Sleep(300 * time.Millisecond)
-	_, err = redisStore.Get(ctx, key)
-	if err == nil {
-		t.Error("value should not be found")
+		consumers++
+		require.Equal(t, value, result.value, "consumed value doesn't match testing data")
 	}
+	require.Equal(t, 1, consumers, "expected exactly one consumer")
+}
+
+func assertGetDelMisses(ctx context.Context, t *testing.T, cacheStore cache.Store, key string) {
+	t.Helper()
+
+	value, found, err := cacheStore.GetDel(ctx, key)
+	require.NoError(t, err, "GetDel on a missing key should not error")
+	require.False(t, found, "GetDel should not find key %q, got value %q", key, value)
+	require.Empty(t, value, "GetDel should return an empty value when not found")
 }