[server, relay] Fix/relay race disconnection (#4174)

Avoid invalid disconnection notifications in case the closed race dials.
In this PR resolve multiple race condition questions. Easier to understand the fix based on commit by commit.

- Remove store dependency from notifier
- Enforce the notification orders
- Fix invalid disconnection notification
- Ensure the order of the events on the consumer side

relay/metrics/realy.go

@@ -20,12 +20,12 @@ type Metrics struct {
 	TransferBytesRecv  metric.Int64Counter
 	AuthenticationTime metric.Float64Histogram
 	PeerStoreTime      metric.Float64Histogram
-
-	peers            metric.Int64UpDownCounter
-	peerActivityChan chan string
-	peerLastActive   map[string]time.Time
-	mutexActivity    sync.Mutex
-	ctx              context.Context
+	peerReconnections  metric.Int64Counter
+	peers              metric.Int64UpDownCounter
+	peerActivityChan   chan string
+	peerLastActive     map[string]time.Time
+	mutexActivity      sync.Mutex
+	ctx                context.Context
 }
 
 func NewMetrics(ctx context.Context, meter metric.Meter) (*Metrics, error) {
@@ -80,6 +80,13 @@ func NewMetrics(ctx context.Context, meter metric.Meter) (*Metrics, error) {
 		return nil, err
 	}
 
+	peerReconnections, err := meter.Int64Counter("relay_peer_reconnections_total",
+		metric.WithDescription("Total number of times peers have reconnected and closed old connections"),
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	m := &Metrics{
 		Meter:              meter,
 		TransferBytesSent:  bytesSent,
@@ -87,6 +94,7 @@ func NewMetrics(ctx context.Context, meter metric.Meter) (*Metrics, error) {
 		AuthenticationTime: authTime,
 		PeerStoreTime:      peerStoreTime,
 		peers:              peers,
+		peerReconnections:  peerReconnections,
 
 		ctx:              ctx,
 		peerActivityChan: make(chan string, 10),
@@ -138,6 +146,10 @@ func (m *Metrics) PeerDisconnected(id string) {
 	delete(m.peerLastActive, id)
 }
 
+func (m *Metrics) RecordPeerReconnection() {
+	m.peerReconnections.Add(m.ctx, 1)
+}
+
 // PeerActivity increases the active connections
 func (m *Metrics) PeerActivity(peerID string) {
 	select {