[management] fix the issue with duplicated peers with the same key (#5053)

This commit is contained in:
Vlad
2026-01-09 11:49:26 +01:00
committed by GitHub
parent 0ad0c81899
commit 684fc0d2a2
13 changed files with 190 additions and 14 deletions

View File

@@ -3465,11 +3465,11 @@ func TestPropagateUserGroupMemberships(t *testing.T) {
account, err := manager.GetOrCreateAccountByUser(ctx, auth.UserAuth{UserId: initiatorId, Domain: domain})
require.NoError(t, err)
peer1 := &nbpeer.Peer{ID: "peer1", AccountID: account.Id, UserID: initiatorId, IP: net.IP{1, 1, 1, 1}, DNSLabel: "peer1.domain.test"}
peer1 := &nbpeer.Peer{ID: "peer1", AccountID: account.Id, Key: "key1", UserID: initiatorId, IP: net.IP{1, 1, 1, 1}, DNSLabel: "peer1.domain.test"}
err = manager.Store.AddPeerToAccount(ctx, peer1)
require.NoError(t, err)
peer2 := &nbpeer.Peer{ID: "peer2", AccountID: account.Id, UserID: initiatorId, IP: net.IP{2, 2, 2, 2}, DNSLabel: "peer2.domain.test"}
peer2 := &nbpeer.Peer{ID: "peer2", AccountID: account.Id, Key: "key2", UserID: initiatorId, IP: net.IP{2, 2, 2, 2}, DNSLabel: "peer2.domain.test"}
err = manager.Store.AddPeerToAccount(ctx, peer2)
require.NoError(t, err)

View File

@@ -893,6 +893,7 @@ func Test_AddPeerAndAddToAll(t *testing.T) {
peer := &peer2.Peer{
ID: strconv.Itoa(i),
AccountID: accountID,
Key: "key" + strconv.Itoa(i),
DNSLabel: "peer" + strconv.Itoa(i),
IP: uint32ToIP(uint32(i)),
}

View File

@@ -404,10 +404,11 @@ func CreateIndexIfNotExists[T any](ctx context.Context, db *gorm.DB, indexName s
if dialect == "mysql" {
var withLength []string
for _, col := range columns {
if col == "ip" || col == "dns_label" {
withLength = append(withLength, fmt.Sprintf("%s(64)", col))
quotedCol := fmt.Sprintf("`%s`", col)
if col == "ip" || col == "dns_label" || col == "key" {
withLength = append(withLength, fmt.Sprintf("%s(64)", quotedCol))
} else {
withLength = append(withLength, col)
withLength = append(withLength, quotedCol)
}
}
columnClause = strings.Join(withLength, ", ")
@@ -487,3 +488,54 @@ func MigrateJsonToTable[T any](ctx context.Context, db *gorm.DB, columnName stri
log.WithContext(ctx).Infof("Migration of JSON field %s from table %s into separate table completed", columnName, tableName)
return nil
}
func RemoveDuplicatePeerKeys(ctx context.Context, db *gorm.DB) error {
if !db.Migrator().HasTable("peers") {
log.WithContext(ctx).Debug("peers table does not exist, skipping duplicate key cleanup")
return nil
}
keyColumn := GetColumnName(db, "key")
var duplicates []struct {
Key string
Count int64
}
if err := db.Table("peers").
Select(keyColumn + ", COUNT(*) as count").
Group(keyColumn).
Having("COUNT(*) > 1").
Find(&duplicates).Error; err != nil {
return fmt.Errorf("find duplicate keys: %w", err)
}
if len(duplicates) == 0 {
return nil
}
log.WithContext(ctx).Warnf("Found %d duplicate peer keys, cleaning up", len(duplicates))
for _, dup := range duplicates {
var peerIDs []string
if err := db.Table("peers").
Select("id").
Where(keyColumn+" = ?", dup.Key).
Order("peer_status_last_seen DESC").
Pluck("id", &peerIDs).Error; err != nil {
return fmt.Errorf("get peers for key: %w", err)
}
if len(peerIDs) <= 1 {
continue
}
idsToDelete := peerIDs[1:]
if err := db.Table("peers").Where("id IN ?", idsToDelete).Delete(nil).Error; err != nil {
return fmt.Errorf("delete duplicate peers: %w", err)
}
}
return nil
}

View File

@@ -340,3 +340,104 @@ func TestCreateIndexIfExists(t *testing.T) {
exist = db.Migrator().HasIndex(&nbpeer.Peer{}, indexName)
assert.True(t, exist, "Should have the index")
}
type testPeer struct {
ID string `gorm:"primaryKey"`
Key string `gorm:"index"`
PeerStatusLastSeen time.Time
PeerStatusConnected bool
}
func (testPeer) TableName() string {
return "peers"
}
func setupPeerTestDB(t *testing.T) *gorm.DB {
t.Helper()
db := setupDatabase(t)
_ = db.Migrator().DropTable(&testPeer{})
err := db.AutoMigrate(&testPeer{})
require.NoError(t, err, "Failed to auto-migrate tables")
return db
}
func TestRemoveDuplicatePeerKeys_NoDuplicates(t *testing.T) {
db := setupPeerTestDB(t)
now := time.Now()
peers := []testPeer{
{ID: "peer1", Key: "key1", PeerStatusLastSeen: now},
{ID: "peer2", Key: "key2", PeerStatusLastSeen: now},
{ID: "peer3", Key: "key3", PeerStatusLastSeen: now},
}
for _, p := range peers {
err := db.Create(&p).Error
require.NoError(t, err)
}
err := migration.RemoveDuplicatePeerKeys(context.Background(), db)
require.NoError(t, err)
var count int64
db.Model(&testPeer{}).Count(&count)
assert.Equal(t, int64(len(peers)), count, "All peers should remain when no duplicates")
}
func TestRemoveDuplicatePeerKeys_WithDuplicates(t *testing.T) {
db := setupPeerTestDB(t)
now := time.Now()
peers := []testPeer{
{ID: "peer1", Key: "key1", PeerStatusLastSeen: now.Add(-2 * time.Hour)},
{ID: "peer2", Key: "key1", PeerStatusLastSeen: now.Add(-1 * time.Hour)},
{ID: "peer3", Key: "key1", PeerStatusLastSeen: now},
{ID: "peer4", Key: "key2", PeerStatusLastSeen: now},
{ID: "peer5", Key: "key3", PeerStatusLastSeen: now.Add(-1 * time.Hour)},
{ID: "peer6", Key: "key3", PeerStatusLastSeen: now},
}
for _, p := range peers {
err := db.Create(&p).Error
require.NoError(t, err)
}
err := migration.RemoveDuplicatePeerKeys(context.Background(), db)
require.NoError(t, err)
var count int64
db.Model(&testPeer{}).Count(&count)
assert.Equal(t, int64(3), count, "Should have 3 peers after removing duplicates")
var remainingPeers []testPeer
err = db.Find(&remainingPeers).Error
require.NoError(t, err)
remainingIDs := make(map[string]bool)
for _, p := range remainingPeers {
remainingIDs[p.ID] = true
}
assert.True(t, remainingIDs["peer3"], "peer3 should remain (most recent for key1)")
assert.True(t, remainingIDs["peer4"], "peer4 should remain (only peer for key2)")
assert.True(t, remainingIDs["peer6"], "peer6 should remain (most recent for key3)")
assert.False(t, remainingIDs["peer1"], "peer1 should be deleted (older duplicate)")
assert.False(t, remainingIDs["peer2"], "peer2 should be deleted (older duplicate)")
assert.False(t, remainingIDs["peer5"], "peer5 should be deleted (older duplicate)")
}
func TestRemoveDuplicatePeerKeys_EmptyTable(t *testing.T) {
db := setupPeerTestDB(t)
err := migration.RemoveDuplicatePeerKeys(context.Background(), db)
require.NoError(t, err, "Should not fail on empty table")
}
func TestRemoveDuplicatePeerKeys_NoTable(t *testing.T) {
db := setupDatabase(t)
_ = db.Migrator().DropTable(&testPeer{})
err := migration.RemoveDuplicatePeerKeys(context.Background(), db)
require.NoError(t, err, "Should not fail when table does not exist")
}

View File

@@ -19,7 +19,7 @@ type Peer struct {
// AccountID is a reference to Account that this object belongs
AccountID string `json:"-" gorm:"index"`
// WireGuard public key
Key string `gorm:"index"`
Key string // uniqueness index (check migrations)
// IP address of the Peer
IP net.IP `gorm:"serializer:json"` // uniqueness index per accountID (check migrations)
// Meta is a Peer system meta data

View File

@@ -2129,12 +2129,14 @@ func Test_DeletePeer(t *testing.T) {
"peer1": {
ID: "peer1",
AccountID: accountID,
Key: "key1",
IP: net.IP{1, 1, 1, 1},
DNSLabel: "peer1.test",
},
"peer2": {
ID: "peer2",
AccountID: accountID,
Key: "key2",
IP: net.IP{2, 2, 2, 2},
DNSLabel: "peer2.test",
},

View File

@@ -968,6 +968,7 @@ func TestSqlite_GetTakenIPs(t *testing.T) {
peer1 := &nbpeer.Peer{
ID: "peer1",
AccountID: existingAccountID,
Key: "key1",
DNSLabel: "peer1",
IP: net.IP{1, 1, 1, 1},
}
@@ -982,6 +983,7 @@ func TestSqlite_GetTakenIPs(t *testing.T) {
peer2 := &nbpeer.Peer{
ID: "peer1second",
AccountID: existingAccountID,
Key: "key2",
DNSLabel: "peer1-1",
IP: net.IP{2, 2, 2, 2},
}
@@ -1009,6 +1011,7 @@ func TestSqlite_GetPeerLabelsInAccount(t *testing.T) {
peer1 := &nbpeer.Peer{
ID: "peer1",
AccountID: existingAccountID,
Key: "key1",
DNSLabel: "peer1",
IP: net.IP{1, 1, 1, 1},
}
@@ -1022,6 +1025,7 @@ func TestSqlite_GetPeerLabelsInAccount(t *testing.T) {
peer2 := &nbpeer.Peer{
ID: "peer1second",
AccountID: existingAccountID,
Key: "key2",
DNSLabel: "peer1-1",
IP: net.IP{2, 2, 2, 2},
}
@@ -1048,6 +1052,7 @@ func Test_AddPeerWithSameDnsLabel(t *testing.T) {
peer1 := &nbpeer.Peer{
ID: "peer1",
AccountID: existingAccountID,
Key: "key1",
DNSLabel: "peer1.domain.test",
}
err = store.AddPeerToAccount(context.Background(), peer1)
@@ -1056,6 +1061,7 @@ func Test_AddPeerWithSameDnsLabel(t *testing.T) {
peer2 := &nbpeer.Peer{
ID: "peer1second",
AccountID: existingAccountID,
Key: "key2",
DNSLabel: "peer1.domain.test",
}
err = store.AddPeerToAccount(context.Background(), peer2)
@@ -1073,6 +1079,7 @@ func Test_AddPeerWithSameIP(t *testing.T) {
peer1 := &nbpeer.Peer{
ID: "peer1",
AccountID: existingAccountID,
Key: "key1",
IP: net.IP{1, 1, 1, 1},
}
err = store.AddPeerToAccount(context.Background(), peer1)
@@ -1081,6 +1088,7 @@ func Test_AddPeerWithSameIP(t *testing.T) {
peer2 := &nbpeer.Peer{
ID: "peer1second",
AccountID: existingAccountID,
Key: "key2",
IP: net.IP{1, 1, 1, 1},
}
err = store.AddPeerToAccount(context.Background(), peer2)
@@ -3696,6 +3704,7 @@ func BenchmarkGetAccountPeers(b *testing.B) {
peer := &nbpeer.Peer{
ID: fmt.Sprintf("peer-%d", i),
AccountID: accountID,
Key: fmt.Sprintf("key-%d", i),
DNSLabel: fmt.Sprintf("peer%d.example.com", i),
IP: intToIPv4(uint32(i)),
}

View File

@@ -350,8 +350,13 @@ func getMigrationsPreAuto(ctx context.Context) []migrationFunc {
func(db *gorm.DB) error {
return migration.MigrateNewField[types.User](ctx, db, "email", "")
},
func(db *gorm.DB) error {
return migration.RemoveDuplicatePeerKeys(ctx, db)
},
}
} // migratePostAuto migrates the SQLite database to the latest schema
}
// migratePostAuto migrates the SQLite database to the latest schema
func migratePostAuto(ctx context.Context, db *gorm.DB) error {
migrations := getMigrationsPostAuto(ctx)
@@ -381,6 +386,12 @@ func getMigrationsPostAuto(ctx context.Context) []migrationFunc {
}
})
},
func(db *gorm.DB) error {
return migration.DropIndex[nbpeer.Peer](ctx, db, "idx_peers_key")
},
func(db *gorm.DB) error {
return migration.CreateIndexIfNotExists[nbpeer.Peer](ctx, db, "idx_peers_key_unique", "key")
},
}
}

View File

@@ -14,7 +14,7 @@ CREATE TABLE `posture_checks` (`id` text,`name` text,`description` text,`account
CREATE TABLE `network_addresses` (`net_ip` text,`mac` text);
CREATE INDEX `idx_accounts_domain` ON `accounts`(`domain`);
CREATE INDEX `idx_setup_keys_account_id` ON `setup_keys`(`account_id`);
CREATE INDEX `idx_peers_key` ON `peers`(`key`);
CREATE UNIQUE INDEX `idx_peers_key_unique` ON `peers`(`key`);
CREATE INDEX `idx_peers_account_id` ON `peers`(`account_id`);
CREATE INDEX `idx_users_account_id` ON `users`(`account_id`);
CREATE INDEX `idx_personal_access_tokens_user_id` ON `personal_access_tokens`(`user_id`);

View File

@@ -18,7 +18,7 @@ CREATE TABLE `network_resources` (`id` text,`network_id` text,`account_id` text,
CREATE TABLE `networks` (`id` text,`account_id` text,`name` text,`description` text,PRIMARY KEY (`id`),CONSTRAINT `fk_accounts_networks` FOREIGN KEY (`account_id`) REFERENCES `accounts`(`id`));
CREATE INDEX `idx_accounts_domain` ON `accounts`(`domain`);
CREATE INDEX `idx_setup_keys_account_id` ON `setup_keys`(`account_id`);
CREATE INDEX `idx_peers_key` ON `peers`(`key`);
CREATE UNIQUE INDEX `idx_peers_key_unique` ON `peers`(`key`);
CREATE INDEX `idx_peers_account_id` ON `peers`(`account_id`);
CREATE INDEX `idx_peers_account_id_ip` ON `peers`(`account_id`,`ip`);
CREATE INDEX `idx_users_account_id` ON `users`(`account_id`);
@@ -54,4 +54,4 @@ INSERT INTO policy_rules VALUES('cs387mkv2d4bgq41b6n0','cs1tnh0hhcjnqoiuebf0','D
INSERT INTO network_routers VALUES('ctc20ji7qv9ck2sebc80','ct286bi7qv930dsrrug0','bf1c8084-ba50-4ce7-9439-34653001fc3b','cs1tnh0hhcjnqoiuebeg',NULL,0,0);
INSERT INTO network_resources VALUES ('ctc4nci7qv9061u6ilfg','ct286bi7qv930dsrrug0','bf1c8084-ba50-4ce7-9439-34653001fc3b','Host','192.168.1.1');
INSERT INTO networks VALUES('ct286bi7qv930dsrrug0','bf1c8084-ba50-4ce7-9439-34653001fc3b','Test Network','Test Network');
INSERT INTO peers VALUES('ct286bi7qv930dsrrug0','bf1c8084-ba50-4ce7-9439-34653001fc3b','','','"192.168.0.0"','','','','','','','','','','','','','','','','','test','test','2023-01-01 00:00:00+00:00',0,0,0,'a23efe53-63fb-11ec-90d6-0242ac120003','',0,0,'2023-01-01 00:00:00+00:00','2023-01-01 00:00:00+00:00',0,'','','',0);
INSERT INTO peers VALUES('ct286bi7qv930dsrrug0','bf1c8084-ba50-4ce7-9439-34653001fc3b','6kjbmVq1hmucVzvBXo5OucY5OYv+jSsB1jUTLq291Do=','','"192.168.0.0"','','','','','','','','','','','','','','','','','test','test','2023-01-01 00:00:00+00:00',0,0,0,'a23efe53-63fb-11ec-90d6-0242ac120003','',0,0,'2023-01-01 00:00:00+00:00','2023-01-01 00:00:00+00:00',0,'','','',0);

View File

@@ -14,7 +14,7 @@ CREATE TABLE `posture_checks` (`id` text,`name` text,`description` text,`account
CREATE TABLE `network_addresses` (`net_ip` text,`mac` text);
CREATE INDEX `idx_accounts_domain` ON `accounts`(`domain`);
CREATE INDEX `idx_setup_keys_account_id` ON `setup_keys`(`account_id`);
CREATE INDEX `idx_peers_key` ON `peers`(`key`);
CREATE UNIQUE INDEX `idx_peers_key_unique` ON `peers`(`key`);
CREATE INDEX `idx_peers_account_id` ON `peers`(`account_id`);
CREATE INDEX `idx_users_account_id` ON `users`(`account_id`);
CREATE INDEX `idx_personal_access_tokens_user_id` ON `personal_access_tokens`(`user_id`);

View File

@@ -14,7 +14,7 @@ CREATE TABLE `posture_checks` (`id` text,`name` text,`description` text,`account
CREATE TABLE `network_addresses` (`net_ip` text,`mac` text);
CREATE INDEX `idx_accounts_domain` ON `accounts`(`domain`);
CREATE INDEX `idx_setup_keys_account_id` ON `setup_keys`(`account_id`);
CREATE INDEX `idx_peers_key` ON `peers`(`key`);
CREATE UNIQUE INDEX `idx_peers_key_unique` ON `peers`(`key`);
CREATE INDEX `idx_peers_account_id` ON `peers`(`account_id`);
CREATE INDEX `idx_users_account_id` ON `users`(`account_id`);
CREATE INDEX `idx_personal_access_tokens_user_id` ON `personal_access_tokens`(`user_id`);
@@ -30,7 +30,7 @@ INSERT INTO setup_keys VALUES('','bf1c8084-ba50-4ce7-9439-34653001fc3b','A2C8E62
INSERT INTO peers VALUES('cfvprsrlo1hqoo49ohog','bf1c8084-ba50-4ce7-9439-34653001fc3b','5rvhvriKJZ3S9oxYToVj5TzDM9u9y8cxg7htIMWlYAg=','72546A29-6BC8-4311-BCFC-9CDBF33F1A48','"100.64.114.31"','f2a34f6a4731','linux','Linux','11','unknown','Debian GNU/Linux','','0.12.0','','',NULL,'','','','{"Cloud":"","Platform":""}',NULL,'f2a34f6a4731','f2a34f6a4731','2023-03-02 09:21:02.189035775+01:00',0,0,0,'','ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILzUUSYG/LGnV8zarb2SGN+tib/PZ+M7cL4WtTzUrTpk',0,1,0,'2023-03-01 19:48:19.817799698+01:00','2024-10-02 17:00:32.527947+02:00',0,'""','','',0);
INSERT INTO peers VALUES('cg05lnblo1hkg2j514p0','bf1c8084-ba50-4ce7-9439-34653001fc3b','RlSy2vzoG2HyMBTUImXOiVhCBiiBa5qD5xzMxkiFDW4=','','"100.64.39.54"','expiredhost','linux','Linux','22.04','x86_64','Ubuntu','','development','','',NULL,'','','','{"Cloud":"","Platform":""}',NULL,'expiredhost','expiredhost','2023-03-02 09:19:57.276717255+01:00',0,1,0,'edafee4e-63fb-11ec-90d6-0242ac120003','ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMbK5ZXJsGOOWoBT4OmkPtgdPZe2Q7bDuS/zjn2CZxhK',0,1,0,'2023-03-02 09:14:21.791679181+01:00','2024-10-02 17:00:32.527947+02:00',0,'""','','',0);
INSERT INTO peers VALUES('cg3161rlo1hs9cq94gdg','bf1c8084-ba50-4ce7-9439-34653001fc3b','mVABSKj28gv+JRsf7e0NEGKgSOGTfU/nPB2cpuG56HU=','','"100.64.117.96"','testhost','linux','Linux','22.04','x86_64','Ubuntu','','development','','',NULL,'','','','{"Cloud":"","Platform":""}',NULL,'testhost','testhost','2023-03-06 18:21:27.252010027+01:00',0,0,0,'edafee4e-63fb-11ec-90d6-0242ac120003','ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINWvvUkFFcrj48CWTkNUb/do/n52i1L5dH4DhGu+4ZuM',0,0,0,'2023-03-07 09:02:47.442857106+01:00','2024-10-02 17:00:32.527947+02:00',0,'""','','',0);
INSERT INTO peers VALUES('csrnkiq7qv9d8aitqd50','bf1c8084-ba50-4ce7-9439-34653001fc3b','mVABSKj28gv+JRsf7e0NEGKgSOGTfU/nPB2cpuG56HU=','','"100.64.117.97"','testhost','linux','Linux','22.04','x86_64','Ubuntu','','development','','',NULL,'','','','{"Cloud":"","Platform":""}',NULL,'testhost','testhost-1','2023-03-06 18:21:27.252010027+01:00',0,0,0,'f4f6d672-63fb-11ec-90d6-0242ac120003','ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINWvvUkFFcrj48CWTkNUb/do/n52i1L5dH4DhGu+4ZuM',0,0,1,'2023-03-07 09:02:47.442857106+01:00','2024-10-02 17:00:32.527947+02:00',0,'""','','',0);
INSERT INTO peers VALUES('csrnkiq7qv9d8aitqd50','bf1c8084-ba50-4ce7-9439-34653001fc3b','nVABSKj28gv+JRsf7e0NEGKgSOGTfU/nPB2cpuG56HX=','','"100.64.117.97"','testhost','linux','Linux','22.04','x86_64','Ubuntu','','development','','',NULL,'','','','{"Cloud":"","Platform":""}',NULL,'testhost','testhost-1','2023-03-06 18:21:27.252010027+01:00',0,0,0,'f4f6d672-63fb-11ec-90d6-0242ac120003','ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINWvvUkFFcrj48CWTkNUb/do/n52i1L5dH4DhGu+4ZuM',0,0,1,'2023-03-07 09:02:47.442857106+01:00','2024-10-02 17:00:32.527947+02:00',0,'""','','',0);
INSERT INTO users VALUES('f4f6d672-63fb-11ec-90d6-0242ac120003','bf1c8084-ba50-4ce7-9439-34653001fc3b','user',0,0,'','[]',0,NULL,'2024-10-02 17:00:32.528196+02:00','api',0,'');
INSERT INTO users VALUES('edafee4e-63fb-11ec-90d6-0242ac120003','bf1c8084-ba50-4ce7-9439-34653001fc3b','admin',0,0,'','[]',0,NULL,'2024-10-02 17:00:32.528196+02:00','api',0,'');
INSERT INTO installations VALUES(1,'');

View File

@@ -14,7 +14,7 @@ CREATE TABLE `posture_checks` (`id` text,`name` text,`description` text,`account
CREATE TABLE `network_addresses` (`net_ip` text,`mac` text);
CREATE INDEX `idx_accounts_domain` ON `accounts`(`domain`);
CREATE INDEX `idx_setup_keys_account_id` ON `setup_keys`(`account_id`);
CREATE INDEX `idx_peers_key` ON `peers`(`key`);
CREATE UNIQUE INDEX `idx_peers_key_unique` ON `peers`(`key`);
CREATE INDEX `idx_peers_account_id` ON `peers`(`account_id`);
CREATE INDEX `idx_users_account_id` ON `users`(`account_id`);
CREATE INDEX `idx_personal_access_tokens_user_id` ON `personal_access_tokens`(`user_id`);