[client] Keep a withdrawal from discarding a payload being delivered

Completing an offer and delivering it ran with no lock held across the two,
while a sender could send DELETE at any point and withdraw ran spool.Remove
without looking at the offer's state. A withdrawal landing in that window
deleted the staged bytes out from under the copy: delivery failed with
"open spooled file: no such file or directory", the transfer was recorded as
failed, and the payload was gone although the sender had seen its upload
succeed.

A completed offer is no longer withdrawable, and publishing or discarding one
offer's payloads now serialises on a per-offer lock the two sinks share, so a
removal waits for a delivery in flight instead of racing it. deliver() drops
the spool as its last step and reaches it through removeLocked, since the
lock it would otherwise retake is the one it already holds.

client/internal/filedrop/sink.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -50,8 +51,19 @@ type PlatformWriter interface {
 	Written() int64
 }
 
+// offerLocks serialises the calls that publish or discard one offer's staged
+// payloads. Delivery copies out of the spool while the sender may still send a
+// withdrawal, and the two arrive on different goroutines through different
+// types, so without this a DELETE landing mid-delivery removes the bytes the
+// copy is reading.
+type offerLocks struct {
+	mu    sync.Mutex
+	locks map[OfferID]*sync.Mutex
+}
+
 // platformSink adapts a PlatformSink to the Sink the receiver uses.
 type platformSink struct {
+	offerLocks
 	platform PlatformSink
 }
 
@@ -68,6 +80,34 @@ func NewPlatformSink(platform PlatformSink) (Sink, error) {
 	return &platformSink{platform: platform}, nil
 }
 
+// lock takes the lock guarding one offer's staged payloads and returns the
+// release. Entries are dropped on release, so the map tracks only offers with
+// a call in flight rather than growing with every offer ever seen.
+func (l *offerLocks) lock(id OfferID) func() {
+	l.mu.Lock()
+	if l.locks == nil {
+		l.locks = make(map[OfferID]*sync.Mutex)
+	}
+	m, ok := l.locks[id]
+	if !ok {
+		m = &sync.Mutex{}
+		l.locks[id] = m
+	}
+	l.mu.Unlock()
+
+	m.Lock()
+	return func() {
+		m.Unlock()
+
+		l.mu.Lock()
+		defer l.mu.Unlock()
+		if m.TryLock() {
+			m.Unlock()
+			delete(l.locks, id)
+		}
+	}
+}
+
 // DestinationLabel reports where the platform delivers payloads.
 func (s *platformSink) DestinationLabel() string {
 	return s.platform.DestinationLabel()
@@ -105,6 +145,8 @@ func (s *platformSink) Write(id OfferID, index int, name string, offset int64, r
 }
 
 func (s *platformSink) Deliver(offer Offer, _ string) ([]string, error) {
+	defer s.lock(offer.ID)()
+
 	delivered, err := s.platform.Deliver(string(offer.ID))
 	if err != nil {
 		return nil, err
@@ -113,6 +155,8 @@ func (s *platformSink) Deliver(offer Offer, _ string) ([]string, error) {
 }
 
 func (s *platformSink) Remove(id OfferID) {
+	defer s.lock(id)()
+
 	s.platform.Remove(string(id))
 }