-

internal/state/apikeys.go

@@ -0,0 +1,93 @@
+package state
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+
+	"github.com/example/ollama-fair-gateway/internal/auth"
+)
+
+type APIKeyStore struct {
+	mu      sync.Mutex
+	file    AtomicJSON
+	records map[string]auth.StoredAPIKey
+}
+
+type apiKeyFile struct {
+	Keys []auth.StoredAPIKey `json:"keys"`
+}
+
+func NewAPIKeyStore(path string) (*APIKeyStore, error) {
+	s := &APIKeyStore{file: AtomicJSON{Path: path, Mode: 0600}, records: map[string]auth.StoredAPIKey{}}
+	var f apiKeyFile
+	if err := s.file.Load(&f); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+	for _, r := range f.Keys {
+		if r.ID != "" {
+			s.records[r.ID] = r
+		}
+	}
+	return s, nil
+}
+func (s *APIKeyStore) Load() ([]auth.StoredAPIKey, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	out := make([]auth.StoredAPIKey, 0, len(s.records))
+	for _, r := range s.records {
+		r.Scopes = append([]string(nil), r.Scopes...)
+		out = append(out, r)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.Before(out[j].CreatedAt) })
+	return out, nil
+}
+func (s *APIKeyStore) Put(r auth.StoredAPIKey) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if _, ok := s.records[r.ID]; ok {
+		return errors.New("API key id already exists")
+	}
+	s.records[r.ID] = r
+	if err := s.saveLocked(); err != nil {
+		delete(s.records, r.ID)
+		return err
+	}
+	return nil
+}
+func (s *APIKeyStore) Delete(id string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	old, ok := s.records[id]
+	if !ok {
+		return nil
+	}
+	delete(s.records, id)
+	if err := s.saveLocked(); err != nil {
+		s.records[id] = old
+		return err
+	}
+	return nil
+}
+func (s *APIKeyStore) saveLocked() error {
+	a := make([]auth.StoredAPIKey, 0, len(s.records))
+	for _, r := range s.records {
+		a = append(a, r)
+	}
+	sort.Slice(a, func(i, j int) bool { return a[i].CreatedAt.Before(a[j].CreatedAt) })
+	return s.file.Save(apiKeyFile{Keys: a})
+}
+func (s *APIKeyStore) Health(context.Context) error {
+	if err := os.MkdirAll(filepath.Dir(s.file.Path), 0750); err != nil {
+		return err
+	}
+	f, err := os.OpenFile(s.file.Path+".health", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	_ = f.Close()
+	return os.Remove(s.file.Path + ".health")
+}

internal/state/atomic.go

@@ -0,0 +1,110 @@
+package state
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// AtomicJSON provides small, durable JSON state files. Writes are staged in
+// the target directory, fsynced, chmodded, then atomically renamed.
+type AtomicJSON struct {
+	Path string
+	Mode os.FileMode
+}
+
+func (f AtomicJSON) Load(out any) error {
+	b, err := os.ReadFile(f.Path)
+	if errors.Is(err, os.ErrNotExist) {
+		return os.ErrNotExist
+	}
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(b, out); err != nil {
+		return fmt.Errorf("decode %s: %w", f.Path, err)
+	}
+	return nil
+}
+
+func (f AtomicJSON) Save(v any) error {
+	if f.Path == "" {
+		return errors.New("state path is empty")
+	}
+	dir := filepath.Dir(f.Path)
+	if err := os.MkdirAll(dir, 0750); err != nil {
+		return err
+	}
+	b, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n')
+	tmp, err := os.CreateTemp(dir, ".state-*.tmp")
+	if err != nil {
+		return err
+	}
+	name := tmp.Name()
+	ok := false
+	defer func() {
+		_ = tmp.Close()
+		if !ok {
+			_ = os.Remove(name)
+		}
+	}()
+	mode := f.Mode
+	if mode == 0 {
+		mode = 0600
+	}
+	if err := tmp.Chmod(mode); err != nil {
+		return err
+	}
+	if _, err := tmp.Write(b); err != nil {
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	// os.Rename replaces an existing regular file on Unix. On Windows the
+	// destination may need to be removed first; keep a .bak so a failed
+	// replacement does not destroy the previous durable state.
+	if err := os.Rename(name, f.Path); err != nil {
+		bak := f.Path + ".bak"
+		_ = os.Remove(bak)
+		if _, statErr := os.Stat(f.Path); statErr == nil {
+			if rerr := os.Rename(f.Path, bak); rerr != nil {
+				return err
+			}
+		}
+		if rerr := os.Rename(name, f.Path); rerr != nil {
+			_ = os.Rename(bak, f.Path)
+			return rerr
+		}
+		_ = os.Remove(bak)
+	}
+	syncDir(dir)
+	ok = true
+	return nil
+}
+
+func syncDir(dir string) {
+	d, err := os.Open(dir)
+	if err != nil {
+		return
+	}
+	_ = d.Sync()
+	_ = d.Close()
+}
+
+func (f AtomicJSON) Delete() error {
+	err := os.Remove(f.Path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	return err
+}

internal/state/config.go

@@ -0,0 +1,77 @@
+package state
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+
+	"github.com/example/ollama-fair-gateway/internal/config"
+)
+
+const bootstrapSecret = "<bootstrap-secret>"
+
+type ConfigStore struct{ file AtomicJSON }
+
+func NewConfigStore(path string) *ConfigStore {
+	return &ConfigStore{file: AtomicJSON{Path: path, Mode: 0600}}
+}
+func (s *ConfigStore) Path() string { return s.file.Path }
+
+// LoadWithBootstrap restores secrets that deliberately remain owned by the
+// startup configuration and are never copied into the persistent UI override.
+func (s *ConfigStore) LoadWithBootstrap(base *config.Config) (*config.Config, error) {
+	b, err := os.ReadFile(s.file.Path)
+	if err != nil {
+		return nil, err
+	}
+	var c config.Config
+	if err := json.Unmarshal(b, &c); err != nil {
+		return nil, err
+	}
+	c.Auth.APIKeys = append([]config.APIKeyConfig(nil), base.Auth.APIKeys...)
+	if c.UI.SessionSecret == "" || c.UI.SessionSecret == bootstrapSecret {
+		c.UI.SessionSecret = base.UI.SessionSecret
+	}
+	if c.UI.OIDC.ClientSecret == "" || c.UI.OIDC.ClientSecret == bootstrapSecret {
+		c.UI.OIDC.ClientSecret = base.UI.OIDC.ClientSecret
+	}
+	merged, err := json.Marshal(c)
+	if err != nil {
+		return nil, err
+	}
+	return config.ParseBytes(merged)
+}
+
+// Save writes a complete config override but strips bootstrap-owned secrets.
+func (s *ConfigStore) Save(c *config.Config) error {
+	b, err := json.Marshal(c)
+	if err != nil {
+		return err
+	}
+	var clean config.Config
+	if err := json.Unmarshal(b, &clean); err != nil {
+		return err
+	}
+	for i := range clean.Auth.APIKeys {
+		clean.Auth.APIKeys[i].Key = bootstrapSecret
+	}
+	if clean.UI.SessionSecret != "" {
+		clean.UI.SessionSecret = bootstrapSecret
+	}
+	if clean.UI.OIDC.ClientSecret != "" {
+		clean.UI.OIDC.ClientSecret = bootstrapSecret
+	}
+	return s.file.Save(&clean)
+}
+func (s *ConfigStore) Delete() error { return s.file.Delete() }
+func (s *ConfigStore) Exists() bool  { _, err := os.Stat(s.file.Path); return err == nil }
+func (s *ConfigStore) LoadIfExists(base *config.Config) (*config.Config, bool, error) {
+	c, err := s.LoadWithBootstrap(base)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, false, nil
+	}
+	if err != nil {
+		return nil, false, err
+	}
+	return c, true, nil
+}

internal/state/modelplacement.go

@@ -0,0 +1,103 @@
+package state
+
+import (
+	"context"
+	"errors"
+	"os"
+	"sync"
+
+	"github.com/example/ollama-fair-gateway/internal/config"
+)
+
+// ModelPlacementStore persists only runtime overrides. Bootstrap rules remain
+// part of config.json and can be restored per worker by deleting an override.
+type ModelPlacementStore struct {
+	mu   sync.RWMutex
+	file AtomicJSON
+	m    map[string]config.ModelPlacementRule
+}
+
+type modelPlacementFile struct {
+	Workers map[string]config.ModelPlacementRule `json:"workers"`
+}
+
+func NewModelPlacementStore(path string) (*ModelPlacementStore, error) {
+	s := &ModelPlacementStore{file: AtomicJSON{Path: path, Mode: 0600}, m: map[string]config.ModelPlacementRule{}}
+	var f modelPlacementFile
+	if err := s.file.Load(&f); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+	for name, rule := range f.Workers {
+		if err := config.ValidateModelPlacementRule(rule); err != nil {
+			return nil, err
+		}
+		s.m[name] = clonePlacementRule(rule)
+	}
+	return s, nil
+}
+
+func (s *ModelPlacementStore) Get(_ context.Context, worker string) (config.ModelPlacementRule, bool, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	r, ok := s.m[worker]
+	return clonePlacementRule(r), ok, nil
+}
+
+func (s *ModelPlacementStore) List(context.Context) (map[string]config.ModelPlacementRule, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	out := make(map[string]config.ModelPlacementRule, len(s.m))
+	for k, v := range s.m {
+		out[k] = clonePlacementRule(v)
+	}
+	return out, nil
+}
+
+func (s *ModelPlacementStore) Put(_ context.Context, worker string, r config.ModelPlacementRule) error {
+	if err := config.ValidateModelPlacementRule(r); err != nil {
+		return err
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	old, had := s.m[worker]
+	s.m[worker] = clonePlacementRule(r)
+	if err := s.saveLocked(); err != nil {
+		if had {
+			s.m[worker] = old
+		} else {
+			delete(s.m, worker)
+		}
+		return err
+	}
+	return nil
+}
+
+func (s *ModelPlacementStore) Delete(_ context.Context, worker string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	old, had := s.m[worker]
+	delete(s.m, worker)
+	if err := s.saveLocked(); err != nil {
+		if had {
+			s.m[worker] = old
+		}
+		return err
+	}
+	return nil
+}
+
+func (s *ModelPlacementStore) Health(context.Context) error { return nil }
+
+func (s *ModelPlacementStore) saveLocked() error {
+	out := make(map[string]config.ModelPlacementRule, len(s.m))
+	for k, v := range s.m {
+		out[k] = clonePlacementRule(v)
+	}
+	return s.file.Save(modelPlacementFile{Workers: out})
+}
+
+func clonePlacementRule(r config.ModelPlacementRule) config.ModelPlacementRule {
+	r.AllowedModels = append([]string(nil), r.AllowedModels...)
+	r.DeniedModels = append([]string(nil), r.DeniedModels...)
+	return r
+}

internal/state/paths.go

@@ -0,0 +1,46 @@
+package state
+
+import (
+	"github.com/example/ollama-fair-gateway/internal/config"
+	"path/filepath"
+)
+
+type Paths struct {
+	DataDir           string
+	Config            string
+	APIKeys           string
+	Policies          string
+	Metrics           string
+	Quota             string
+	WorkerPerformance string
+	ModelPlacement    string
+	WorkerState       string
+	AutoTune          string
+	WarmModels        string
+	Alerts            string
+	Conversations     string
+	BatchJobs         string
+	BatchDir          string
+	UsageDir          string
+}
+
+func Resolve(c config.StorageConfig) Paths {
+	return Paths{
+		DataDir:           c.DataDir,
+		Config:            filepath.Join(c.DataDir, c.ConfigFile),
+		APIKeys:           filepath.Join(c.DataDir, c.APIKeysFile),
+		Policies:          filepath.Join(c.DataDir, c.PoliciesFile),
+		Metrics:           filepath.Join(c.DataDir, c.MetricsFile),
+		Quota:             filepath.Join(c.DataDir, c.QuotaFile),
+		WorkerPerformance: filepath.Join(c.DataDir, c.WorkerPerformanceFile),
+		ModelPlacement:    filepath.Join(c.DataDir, c.ModelPlacementFile),
+		WorkerState:       filepath.Join(c.DataDir, c.WorkerStateFile),
+		AutoTune:          filepath.Join(c.DataDir, c.AutoTuneFile),
+		WarmModels:        filepath.Join(c.DataDir, c.WarmModelsFile),
+		Alerts:            filepath.Join(c.DataDir, c.AlertsFile),
+		Conversations:     filepath.Join(c.DataDir, c.ConversationsFile),
+		BatchJobs:         filepath.Join(c.DataDir, c.BatchJobsFile),
+		BatchDir:          filepath.Join(c.DataDir, c.BatchJobsDir),
+		UsageDir:          filepath.Join(c.DataDir, "usage"),
+	}
+}

internal/state/policies.go

@@ -0,0 +1,83 @@
+package state
+
+import (
+	"context"
+	"errors"
+	"os"
+	"sync"
+
+	"github.com/example/ollama-fair-gateway/internal/config"
+)
+
+type PolicyStore struct {
+	mu   sync.RWMutex
+	file AtomicJSON
+	m    map[string]config.TenantPolicy
+}
+
+type policyFile struct {
+	Policies map[string]config.TenantPolicy `json:"policies"`
+}
+
+func NewPolicyStore(path string) (*PolicyStore, error) {
+	s := &PolicyStore{file: AtomicJSON{Path: path, Mode: 0600}, m: map[string]config.TenantPolicy{}}
+	var f policyFile
+	if err := s.file.Load(&f); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+	for k, v := range f.Policies {
+		s.m[k] = v
+	}
+	return s, nil
+}
+func (s *PolicyStore) Get(_ context.Context, tenant string) (config.TenantPolicy, bool, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	p, ok := s.m[tenant]
+	return p, ok, nil
+}
+func (s *PolicyStore) Put(_ context.Context, tenant string, p config.TenantPolicy) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	old, had := s.m[tenant]
+	s.m[tenant] = p
+	if err := s.saveLocked(); err != nil {
+		if had {
+			s.m[tenant] = old
+		} else {
+			delete(s.m, tenant)
+		}
+		return err
+	}
+	return nil
+}
+func (s *PolicyStore) Delete(_ context.Context, tenant string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	old, had := s.m[tenant]
+	delete(s.m, tenant)
+	if err := s.saveLocked(); err != nil {
+		if had {
+			s.m[tenant] = old
+		}
+		return err
+	}
+	return nil
+}
+func (s *PolicyStore) List(context.Context) (map[string]config.TenantPolicy, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	out := make(map[string]config.TenantPolicy, len(s.m))
+	for k, v := range s.m {
+		out[k] = v
+	}
+	return out, nil
+}
+func (s *PolicyStore) Health(context.Context) error { return nil }
+func (s *PolicyStore) saveLocked() error {
+	out := make(map[string]config.TenantPolicy, len(s.m))
+	for k, v := range s.m {
+		out[k] = v
+	}
+	return s.file.Save(policyFile{Policies: out})
+}

internal/state/state_test.go

@@ -0,0 +1,180 @@
+package state
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/example/ollama-fair-gateway/internal/auth"
+	"github.com/example/ollama-fair-gateway/internal/config"
+)
+
+func TestPersistentAPIKeySurvivesAuthenticatorRestart(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "keys.json")
+	store, err := NewAPIKeyStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, err := auth.NewWithRuntimeStore(context.Background(), config.AuthConfig{}, store)
+	if err != nil {
+		t.Fatal(err)
+	}
+	info, secret, err := a.CreateAPIKey(auth.APIKeyCreate{Name: "openwebui", Tenant: "interactive", Application: "openwebui"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Source != "persistent" {
+		t.Fatalf("source=%q", info.Source)
+	}
+
+	store2, err := NewAPIKeyStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := auth.NewWithRuntimeStore(context.Background(), config.AuthConfig{}, store2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := httptest.NewRequest("GET", "http://gateway/api/tags", nil)
+	r.Header.Set("Authorization", "Bearer "+secret)
+	id, err := b.Authenticate(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id.Tenant != "interactive" || id.Application != "openwebui" {
+		t.Fatalf("identity=%#v", id)
+	}
+	if _, ok, err := b.DeleteAPIKey(info.ID); err != nil || !ok {
+		t.Fatalf("delete ok=%v err=%v", ok, err)
+	}
+
+	store3, err := NewAPIKeyStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c, err := auth.NewWithRuntimeStore(context.Background(), config.AuthConfig{}, store3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := c.Authenticate(r); err == nil {
+		t.Fatal("deleted key authenticated after restart")
+	}
+}
+
+func TestPersistentPolicySurvivesRestart(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "policies.json")
+	a, err := NewPolicyStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := config.TenantPolicy{TenantWeight: 2, ActorWeight: 3, TenantCreditsPerMinute: 0, ActorCreditsPerMinute: 120}
+	if err := a.Put(context.Background(), "team", want); err != nil {
+		t.Fatal(err)
+	}
+	b, err := NewPolicyStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, ok, err := b.Get(context.Background(), "team")
+	if err != nil || !ok {
+		t.Fatalf("get ok=%v err=%v", ok, err)
+	}
+	if got != want {
+		t.Fatalf("got=%#v want=%#v", got, want)
+	}
+}
+
+func TestConfigStoreKeepsBootstrapSecretsOutOfPersistentFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "gateway-config.json")
+	base := &config.Config{
+		Auth:    config.AuthConfig{APIKeys: []config.APIKeyConfig{{Name: "admin", Key: "super-secret", Tenant: "ops"}}},
+		Workers: []config.WorkerConfig{{Name: "w", URL: "http://127.0.0.1:11434", MaxConcurrent: 1}},
+		UI:      config.UIConfig{SessionSecret: "01234567890123456789012345678901", OIDC: config.UIOIDCConfig{ClientSecret: "oidc-secret"}},
+	}
+	// Fill defaults/validation through the normal parser so the stored object is realistic.
+	b, _ := json.Marshal(base)
+	parsed, err := config.ParseBytes(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	store := NewConfigStore(path)
+	if err := store.Save(parsed); err != nil {
+		t.Fatal(err)
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Contains(raw, []byte("super-secret")) || bytes.Contains(raw, []byte("oidc-secret")) || bytes.Contains(raw, []byte("01234567890123456789012345678901")) {
+		t.Fatalf("persistent config leaked bootstrap secret: %s", raw)
+	}
+	loaded, err := store.LoadWithBootstrap(parsed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if loaded.Auth.APIKeys[0].Key != "super-secret" || loaded.UI.SessionSecret != parsed.UI.SessionSecret || loaded.UI.OIDC.ClientSecret != "oidc-secret" {
+		t.Fatalf("secrets not restored: %#v", loaded.UI)
+	}
+}
+
+func TestModelPlacementStoreSurvivesRestart(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "model-placement.json")
+	a, err := NewModelPlacementStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := config.ModelPlacementRule{Mode: "whitelist", AllowedModels: []string{"qwen3:8b", "gemma4:*"}, DeniedModels: []string{"gemma4:e4b"}}
+	if err := a.Put(context.Background(), "rtx-4090", want); err != nil {
+		t.Fatal(err)
+	}
+	b, err := NewModelPlacementStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, ok, err := b.Get(context.Background(), "rtx-4090")
+	if err != nil || !ok {
+		t.Fatalf("get ok=%v err=%v", ok, err)
+	}
+	if got.Mode != want.Mode || len(got.AllowedModels) != 2 || len(got.DeniedModels) != 1 {
+		t.Fatalf("got=%#v want=%#v", got, want)
+	}
+	if err := b.Delete(context.Background(), "rtx-4090"); err != nil {
+		t.Fatal(err)
+	}
+	c, err := NewModelPlacementStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok, _ := c.Get(context.Background(), "rtx-4090"); ok {
+		t.Fatal("deleted placement override survived restart")
+	}
+}
+
+func TestWorkerRuntimeStorePersistence(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "worker-state.json")
+	s, err := NewWorkerRuntimeStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Put(context.Background(), "node-2", "draining"); err != nil {
+		t.Fatal(err)
+	}
+	s2, err := NewWorkerRuntimeStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m, err := s2.List(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m["node-2"] != "draining" {
+		t.Fatalf("modes=%v", m)
+	}
+	if err := s2.Delete(context.Background(), "node-2"); err != nil {
+		t.Fatal(err)
+	}
+}

internal/state/workerstate.go

@@ -0,0 +1,95 @@
+package state
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+)
+
+type WorkerRuntimeStore struct {
+	mu    sync.Mutex
+	file  AtomicJSON
+	modes map[string]string
+}
+
+type workerRuntimeFile struct {
+	Workers map[string]string `json:"workers"`
+}
+
+func NewWorkerRuntimeStore(path string) (*WorkerRuntimeStore, error) {
+	s := &WorkerRuntimeStore{file: AtomicJSON{Path: path, Mode: 0600}, modes: map[string]string{}}
+	var f workerRuntimeFile
+	if err := s.file.Load(&f); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+	for k, v := range f.Workers {
+		if k != "" {
+			s.modes[k] = v
+		}
+	}
+	return s, nil
+}
+func (s *WorkerRuntimeStore) List(context.Context) (map[string]string, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	out := map[string]string{}
+	for k, v := range s.modes {
+		out[k] = v
+	}
+	return out, nil
+}
+func (s *WorkerRuntimeStore) Put(_ context.Context, name, mode string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	old, had := s.modes[name]
+	s.modes[name] = mode
+	if err := s.saveLocked(); err != nil {
+		if had {
+			s.modes[name] = old
+		} else {
+			delete(s.modes, name)
+		}
+		return err
+	}
+	return nil
+}
+func (s *WorkerRuntimeStore) Delete(_ context.Context, name string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	old, had := s.modes[name]
+	delete(s.modes, name)
+	if err := s.saveLocked(); err != nil {
+		if had {
+			s.modes[name] = old
+		}
+		return err
+	}
+	return nil
+}
+func (s *WorkerRuntimeStore) saveLocked() error {
+	keys := make([]string, 0, len(s.modes))
+	for k := range s.modes {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	m := map[string]string{}
+	for _, k := range keys {
+		m[k] = s.modes[k]
+	}
+	return s.file.Save(workerRuntimeFile{Workers: m})
+}
+func (s *WorkerRuntimeStore) Health(context.Context) error {
+	if err := os.MkdirAll(filepath.Dir(s.file.Path), 0750); err != nil {
+		return err
+	}
+	f, err := os.OpenFile(s.file.Path+".health", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	_ = f.Close()
+	return os.Remove(s.file.Path + ".health")
+}
+func (s *WorkerRuntimeStore) Path() string { return s.file.Path }