Vollständiges Redesign

internal/api/server.go

@@ -0,0 +1,187 @@
+package api
+
+import (
+	"context"
+	"crypto/subtle"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"example.com/siem-greenfield/internal/clickhouse"
+	"example.com/siem-greenfield/internal/config"
+	"example.com/siem-greenfield/internal/postgres"
+)
+
+func Run(ctx context.Context, cfg config.Config) error {
+	pg, e := postgres.Open(ctx, cfg.PostgresURL)
+	if e != nil {
+		return e
+	}
+	defer pg.Close()
+	ch := clickhouse.New(cfg)
+	tpl, e := template.ParseFiles("/app/web/index.html")
+	if e != nil {
+		return e
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { j(w, 200, map[string]string{"status": "ok"}) })
+	mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
+		cctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
+		defer cancel()
+		if e := pg.Pool.Ping(cctx); e != nil {
+			j(w, 503, map[string]string{"status": "not_ready", "component": "postgres"})
+			return
+		}
+		if e := ch.Exec(cctx, "SELECT 1"); e != nil {
+			j(w, 503, map[string]string{"status": "not_ready", "component": "clickhouse"})
+			return
+		}
+		j(w, 200, map[string]string{"status": "ready"})
+	})
+	mux.HandleFunc("/ui", func(w http.ResponseWriter, r *http.Request) {
+		_ = tpl.Execute(w, map[string]string{"Tenant": cfg.TenantID})
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/" {
+			http.Redirect(w, r, "/ui", http.StatusFound)
+			return
+		}
+		http.NotFound(w, r)
+	})
+	mux.HandleFunc("/api/summary", func(w http.ResponseWriter, r *http.Request) { summary(w, r, cfg, pg, ch) })
+	mux.HandleFunc("/api/events", func(w http.ResponseWriter, r *http.Request) { events(w, r, cfg, ch) })
+	mux.HandleFunc("/api/detections", func(w http.ResponseWriter, r *http.Request) { detections(w, r, cfg, pg) })
+	mux.HandleFunc("/api/detections/status", func(w http.ResponseWriter, r *http.Request) { detectionStatus(w, r, cfg, pg) })
+	mux.HandleFunc("/api/agents", func(w http.ResponseWriter, r *http.Request) {
+		x, e := pg.ListAgents(r.Context(), cfg.TenantID)
+		if e != nil {
+			j(w, 500, map[string]string{"error": e.Error()})
+			return
+		}
+		j(w, 200, x)
+	})
+	srv := &http.Server{Addr: cfg.ServiceAddr, Handler: security(basicAuth(mux, cfg.UIUsername, cfg.UIPassword)), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 30 * time.Second, IdleTimeout: 60 * time.Second}
+	go func() {
+		<-ctx.Done()
+		c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(c)
+	}()
+	log.Printf("api/ui listening on %s", cfg.ServiceAddr)
+	e = srv.ListenAndServe()
+	if e == http.ErrServerClosed {
+		return nil
+	}
+	return e
+}
+func summary(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) {
+	ctx, cancel := context.WithTimeout(r.Context(), 8*time.Second)
+	defer cancel()
+	q := fmt.Sprintf(`SELECT uniqExactMerge(cnt_state) events_24h, uniqExact(host_name) active_hosts FROM siem.events_5m WHERE tenant_id=%s AND bucket>=now()-INTERVAL 24 HOUR`, clickhouse.Q(cfg.TenantID))
+	rows, e := ch.QueryJSON(ctx, q)
+	if e != nil {
+		j(w, 500, map[string]string{"error": e.Error()})
+		return
+	}
+	out := map[string]any{"events_24h": 0, "active_hosts": 0}
+	if len(rows) > 0 {
+		for k, v := range rows[0] {
+			out[k] = v
+		}
+	}
+	dc, _ := pg.DetectionCounts(ctx, cfg.TenantID)
+	out["detections"] = dc
+	j(w, 200, out)
+}
+func events(w http.ResponseWriter, r *http.Request, cfg config.Config, ch *clickhouse.Client) {
+	limit := cfg.UIQueryLimit
+	if n, e := strconv.Atoi(r.URL.Query().Get("limit")); e == nil && n > 0 && n <= 2000 {
+		limit = n
+	}
+	hours := 24
+	if n, e := strconv.Atoi(r.URL.Query().Get("hours")); e == nil && n > 0 && n <= 2160 {
+		hours = n
+	}
+	where := []string{"tenant_id=" + clickhouse.Q(cfg.TenantID), fmt.Sprintf("event_time>=now()-INTERVAL %d HOUR", hours)}
+	for key, col := range map[string]string{"host": "host_name", "user": "user_name", "ip": "source_ip", "channel": "channel", "action": "action"} {
+		if v := strings.TrimSpace(r.URL.Query().Get(key)); v != "" {
+			where = append(where, col+"="+clickhouse.Q(v))
+		}
+	}
+	if v := strings.TrimSpace(r.URL.Query().Get("event_code")); v != "" {
+		if _, e := strconv.ParseUint(v, 10, 32); e == nil {
+			where = append(where, "event_code="+v)
+		}
+	}
+	q := fmt.Sprintf(`SELECT event_uid,event_time,host_name,channel,event_code,category,action,outcome,severity,user_name,subject_user,target_user,source_ip,workstation,process_path,message,raw_object_key,raw_index FROM siem.events WHERE %s ORDER BY event_time DESC, ingest_time DESC LIMIT 1 BY event_uid LIMIT %d`, strings.Join(where, " AND "), limit)
+	ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
+	defer cancel()
+	rows, e := ch.QueryJSON(ctx, q)
+	if e != nil {
+		j(w, 500, map[string]string{"error": e.Error()})
+		return
+	}
+	j(w, 200, rows)
+}
+func detections(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) {
+	x, e := pg.ListDetections(r.Context(), cfg.TenantID, 500)
+	if e != nil {
+		j(w, 500, map[string]string{"error": e.Error()})
+		return
+	}
+	j(w, 200, x)
+}
+func detectionStatus(w http.ResponseWriter, r *http.Request, cfg config.Config, pg *postgres.Store) {
+	if r.Method != http.MethodPost {
+		j(w, 405, map[string]string{"error": "method not allowed"})
+		return
+	}
+	var v struct {
+		ID     int64  `json:"id"`
+		Status string `json:"status"`
+	}
+	if e := json.NewDecoder(r.Body).Decode(&v); e != nil {
+		j(w, 400, map[string]string{"error": "invalid json"})
+		return
+	}
+	if e := pg.UpdateDetectionStatus(r.Context(), cfg.TenantID, v.ID, v.Status); e != nil {
+		j(w, 400, map[string]string{"error": e.Error()})
+		return
+	}
+	j(w, 200, map[string]string{"status": "ok"})
+}
+func j(w http.ResponseWriter, s int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(s)
+	_ = json.NewEncoder(w).Encode(v)
+}
+func basicAuth(next http.Handler, user, pass string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/healthz" || r.URL.Path == "/readyz" {
+			next.ServeHTTP(w, r)
+			return
+		}
+		u, p, ok := r.BasicAuth()
+		if !ok || subtle.ConstantTimeCompare([]byte(u), []byte(user)) != 1 || subtle.ConstantTimeCompare([]byte(p), []byte(pass)) != 1 {
+			w.Header().Set("WWW-Authenticate", `Basic realm="Greenfield SIEM", charset="UTF-8"`)
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func security(n http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		w.Header().Set("X-Frame-Options", "DENY")
+		w.Header().Set("Referrer-Policy", "no-referrer")
+		w.Header().Set("Content-Security-Policy", "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'")
+		n.ServeHTTP(w, r)
+	})
+}

internal/clickhouse/client.go

@@ -0,0 +1,98 @@
+package clickhouse
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"example.com/siem-greenfield/internal/config"
+	"example.com/siem-greenfield/internal/contracts"
+)
+
+type Client struct {
+	base, user, pass string
+	hc               *http.Client
+}
+
+func New(cfg config.Config) *Client {
+	return &Client{base: cfg.ClickHouseURL, user: cfg.ClickHouseUser, pass: cfg.ClickHousePassword, hc: &http.Client{Timeout: 30 * time.Second}}
+}
+func (c *Client) request(ctx context.Context, query string, body io.Reader) (*http.Response, error) {
+	u := c.base + "/?query=" + url.QueryEscape(query)
+	req, e := http.NewRequestWithContext(ctx, http.MethodPost, u, body)
+	if e != nil {
+		return nil, e
+	}
+	req.SetBasicAuth(c.user, c.pass)
+	return c.hc.Do(req)
+}
+func (c *Client) Exec(ctx context.Context, q string) error {
+	r, e := c.request(ctx, q, nil)
+	if e != nil {
+		return e
+	}
+	defer r.Body.Close()
+	if r.StatusCode/100 != 2 {
+		b, _ := io.ReadAll(io.LimitReader(r.Body, 8192))
+		return fmt.Errorf("clickhouse %s: %s", r.Status, string(b))
+	}
+	return nil
+}
+func (c *Client) InsertEvents(ctx context.Context, events []contracts.CanonicalEvent) error {
+	if len(events) == 0 {
+		return nil
+	}
+	var b bytes.Buffer
+	enc := json.NewEncoder(&b)
+	enc.SetEscapeHTML(false)
+	for i := range events {
+		if e := enc.Encode(events[i]); e != nil {
+			return e
+		}
+	}
+	q := `INSERT INTO siem.events FORMAT JSONEachRow`
+	r, e := c.request(ctx, q, &b)
+	if e != nil {
+		return e
+	}
+	defer r.Body.Close()
+	if r.StatusCode/100 != 2 {
+		x, _ := io.ReadAll(io.LimitReader(r.Body, 16384))
+		return fmt.Errorf("clickhouse insert %s: %s", r.Status, string(x))
+	}
+	return nil
+}
+func (c *Client) QueryJSON(ctx context.Context, q string) ([]map[string]any, error) {
+	if !strings.Contains(strings.ToUpper(q), "FORMAT") {
+		q += " FORMAT JSONEachRow"
+	}
+	r, e := c.request(ctx, q, nil)
+	if e != nil {
+		return nil, e
+	}
+	defer r.Body.Close()
+	if r.StatusCode/100 != 2 {
+		x, _ := io.ReadAll(io.LimitReader(r.Body, 16384))
+		return nil, fmt.Errorf("clickhouse query %s: %s", r.Status, string(x))
+	}
+	var out []map[string]any
+	s := bufio.NewScanner(r.Body)
+	buf := make([]byte, 0, 64*1024)
+	s.Buffer(buf, 4*1024*1024)
+	for s.Scan() {
+		var m map[string]any
+		if e := json.Unmarshal(s.Bytes(), &m); e != nil {
+			return nil, e
+		}
+		out = append(out, m)
+	}
+	return out, s.Err()
+}
+func Q(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" }

internal/config/config.go

@@ -0,0 +1,103 @@
+package config
+
+import (
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Config struct {
+	ServiceAddr        string
+	PostgresURL        string
+	ClickHouseURL      string
+	ClickHouseUser     string
+	ClickHousePassword string
+	KafkaBrokers       []string
+	KafkaTopic         string
+	KafkaGroup         string
+	EnrollmentKey      string
+	TenantID           string
+	MaxBodyBytes       int64
+	MaxBatchEvents     int
+	RawArchiveEnabled  bool
+	RawSpoolDir        string
+	RawRetention       time.Duration
+	EventRetention     time.Duration
+	DetectorInterval   time.Duration
+	DetectorLookback   time.Duration
+	UIQueryLimit       int
+	UIUsername         string
+	UIPassword         string
+}
+
+func Load() Config {
+	return Config{
+		ServiceAddr:        env("SERVICE_ADDR", ":8080"),
+		PostgresURL:        env("POSTGRES_URL", "postgres://siem:siem@postgres:5432/siem?sslmode=disable"),
+		ClickHouseURL:      strings.TrimRight(env("CLICKHOUSE_URL", "http://clickhouse:8123"), "/"),
+		ClickHouseUser:     env("CLICKHOUSE_USER", "siem"),
+		ClickHousePassword: env("CLICKHOUSE_PASSWORD", "siem"),
+		KafkaBrokers:       splitCSV(env("KAFKA_BROKERS", "redpanda:9092")),
+		KafkaTopic:         env("KAFKA_TOPIC", "siem-events"),
+		KafkaGroup:         env("KAFKA_GROUP", "siem-processor-v1"),
+		EnrollmentKey:      env("ENROLLMENT_KEY", "change-me"),
+		TenantID:           env("TENANT_ID", "default"),
+		MaxBodyBytes:       int64(envInt("MAX_BODY_BYTES", 8*1024*1024)),
+		MaxBatchEvents:     envInt("MAX_BATCH_EVENTS", 1000),
+		RawArchiveEnabled:  envBool("RAW_ARCHIVE_ENABLED", true),
+		RawSpoolDir:        env("RAW_SPOOL_DIR", "/var/spool/siem-raw"),
+		RawRetention:       envDuration("RAW_RETENTION", 30*24*time.Hour),
+		EventRetention:     envDuration("EVENT_RETENTION", 90*24*time.Hour),
+		DetectorInterval:   envDuration("DETECTOR_INTERVAL", 30*time.Second),
+		DetectorLookback:   envDuration("DETECTOR_LOOKBACK", 20*time.Minute),
+		UIQueryLimit:       envInt("UI_QUERY_LIMIT", 500),
+		UIUsername:         env("UI_USERNAME", "admin"),
+		UIPassword:         env("UI_PASSWORD", "change-me"),
+	}
+}
+
+func env(k, d string) string {
+	if v := strings.TrimSpace(os.Getenv(k)); v != "" {
+		return v
+	}
+	return d
+}
+func envInt(k string, d int) int {
+	v := strings.TrimSpace(os.Getenv(k))
+	if v == "" {
+		return d
+	}
+	n, e := strconv.Atoi(v)
+	if e != nil {
+		return d
+	}
+	return n
+}
+func envBool(k string, d bool) bool {
+	v := strings.ToLower(strings.TrimSpace(os.Getenv(k)))
+	if v == "" {
+		return d
+	}
+	return v == "1" || v == "true" || v == "yes" || v == "on"
+}
+func envDuration(k string, d time.Duration) time.Duration {
+	v := strings.TrimSpace(os.Getenv(k))
+	if v == "" {
+		return d
+	}
+	x, e := time.ParseDuration(v)
+	if e != nil {
+		return d
+	}
+	return x
+}
+func splitCSV(v string) []string {
+	var out []string
+	for _, p := range strings.Split(v, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}

internal/contracts/event.go

@@ -0,0 +1,94 @@
+package contracts
+
+import "time"
+
+type EventMetadataPayload struct {
+	Computer              string `json:"computer,omitempty"`
+	ProviderName          string `json:"provider,omitempty"`
+	TargetUser            string `json:"target_user,omitempty"`
+	TargetDomain          string `json:"target_domain,omitempty"`
+	SubjectUser           string `json:"subject_user,omitempty"`
+	SubjectDomain         string `json:"subject_domain,omitempty"`
+	Workstation           string `json:"workstation,omitempty"`
+	Device                string `json:"device,omitempty"`
+	SrcIP                 string `json:"src_ip,omitempty"`
+	SrcPort               string `json:"src_port,omitempty"`
+	DstIP                 string `json:"dst_ip,omitempty"`
+	DstPort               string `json:"dst_port,omitempty"`
+	LogonType             string `json:"logon_type,omitempty"`
+	ProcessName           string `json:"process_name,omitempty"`
+	ParentProcessName     string `json:"parent_process_name,omitempty"`
+	CommandLine           string `json:"command_line,omitempty"`
+	AuthenticationPackage string `json:"authentication_package,omitempty"`
+	LogonProcess          string `json:"logon_process,omitempty"`
+	StatusText            string `json:"status,omitempty"`
+	SubStatusText         string `json:"sub_status,omitempty"`
+	FailureReason         string `json:"failure_reason,omitempty"`
+}
+
+type LogPayload struct {
+	Hostname string                `json:"host"`
+	Channel  string                `json:"channel"`
+	EventID  uint32                `json:"id"`
+	Source   string                `json:"source"`
+	Time     time.Time             `json:"ts"`
+	Message  string                `json:"msg,omitempty"`
+	Metadata *EventMetadataPayload `json:"meta,omitempty"`
+}
+
+type IngestEnvelope struct {
+	Version    int          `json:"version"`
+	TenantID   string       `json:"tenant_id"`
+	AgentID    string       `json:"agent_id"`
+	BatchUID   string       `json:"batch_uid"`
+	RemoteIP   string       `json:"remote_ip"`
+	ReceivedAt time.Time    `json:"received_at"`
+	Events     []LogPayload `json:"events"`
+}
+
+type CanonicalEvent struct {
+	EventUID          string            `json:"event_uid"`
+	QueuePartition    int32             `json:"queue_partition"`
+	QueueOffset       int64             `json:"queue_offset"`
+	TenantID          string            `json:"tenant_id"`
+	EventTime         string            `json:"event_time"`
+	IngestTime        string            `json:"ingest_time"`
+	AgentID           string            `json:"agent_id"`
+	HostName          string            `json:"host_name"`
+	SourceType        string            `json:"source_type"`
+	Channel           string            `json:"channel"`
+	Provider          string            `json:"provider"`
+	EventCode         uint32            `json:"event_code"`
+	Category          string            `json:"category"`
+	Action            string            `json:"action"`
+	Outcome           string            `json:"outcome"`
+	Severity          uint8             `json:"severity"`
+	UserName          string            `json:"user_name"`
+	UserDomain        string            `json:"user_domain"`
+	SubjectUser       string            `json:"subject_user"`
+	SubjectDomain     string            `json:"subject_domain"`
+	TargetUser        string            `json:"target_user"`
+	TargetDomain      string            `json:"target_domain"`
+	SourceIP          string            `json:"source_ip"`
+	SourcePort        uint16            `json:"source_port"`
+	DestinationIP     string            `json:"destination_ip"`
+	DestinationPort   uint16            `json:"destination_port"`
+	Workstation       string            `json:"workstation"`
+	LogonType         string            `json:"logon_type"`
+	AuthenticationPackage string        `json:"authentication_package"`
+	LogonProcess      string            `json:"logon_process"`
+	StatusCode        string            `json:"status_code"`
+	SubStatusCode     string            `json:"sub_status_code"`
+	FailureReason     string            `json:"failure_reason"`
+	ProcessPath       string            `json:"process_path"`
+	ParentProcessPath string            `json:"parent_process_path"`
+	CommandLine       string            `json:"command_line"`
+	Message           string            `json:"message"`
+	Attributes        map[string]string `json:"attributes"`
+	RawObjectKey      string            `json:"raw_object_key"`
+	RawIndex          uint32            `json:"raw_index"`
+	PayloadHash       string            `json:"payload_hash"`
+	SchemaVersion     uint16            `json:"schema_version"`
+	ParserVersion     uint16            `json:"parser_version"`
+	IngestDelayMS     int64             `json:"ingest_delay_ms"`
+}

internal/detector/detector.go

@@ -0,0 +1,152 @@
+package detector
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+	"time"
+
+	"example.com/siem-greenfield/internal/clickhouse"
+	"example.com/siem-greenfield/internal/config"
+	"example.com/siem-greenfield/internal/postgres"
+)
+
+type rule struct {
+	name, severity string
+	eventCode      uint32
+	score          float64
+	query          func(time.Time, time.Time, string) string
+	summary        func(map[string]any) string
+}
+
+func Run(ctx context.Context, cfg config.Config) error {
+	pg, e := postgres.Open(ctx, cfg.PostgresURL)
+	if e != nil {
+		return e
+	}
+	defer pg.Close()
+	ch := clickhouse.New(cfg)
+	ticker := time.NewTicker(cfg.DetectorInterval)
+	defer ticker.Stop()
+	run := func() {
+		if e := runAll(ctx, cfg, pg, ch); e != nil {
+			log.Printf("detector cycle: %v", e)
+		}
+	}
+	run()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			run()
+		}
+	}
+}
+func runAll(ctx context.Context, cfg config.Config, pg *postgres.Store, ch *clickhouse.Client) error {
+	end := time.Now().UTC()
+	start := end.Add(-cfg.DetectorLookback)
+	for _, r := range rules() {
+		q := r.query(start, end, cfg.TenantID)
+		rows, e := ch.QueryJSON(ctx, q)
+		if e != nil {
+			return fmt.Errorf("%s: %w", r.name, e)
+		}
+		for _, row := range rows {
+			host := str(row["host_name"])
+			user := str(row["user_name"])
+			ip := str(row["source_ip"])
+			workstation := str(row["workstation"])
+			count := int64(num(row["cnt"]))
+			ws := timeVal(row["window_start"], start)
+			we := timeVal(row["window_end"], end)
+			fp := fingerprint(r.name, host, user, ip, workstation, strconv.FormatInt(ws.Unix()/300, 10))
+			d := postgres.Detection{Fingerprint: fp, RuleName: r.name, Severity: r.severity, Hostname: host, UserName: user, SourceIP: ip, Workstation: workstation, EventCode: r.eventCode, Score: r.score, WindowStart: ws, WindowEnd: we, Summary: r.summary(row), Count: max64(1, count)}
+			if e := pg.UpsertDetection(ctx, d, cfg.TenantID); e != nil {
+				return e
+			}
+		}
+	}
+	return nil
+}
+func rules() []rule {
+	return []rule{
+		{name: "audit_log_cleared", severity: "critical", eventCode: 1102, score: 9.8, query: simpleEvent(1102, 1), summary: func(m map[string]any) string {
+			return fmt.Sprintf("Audit-Log auf %s wurde gelöscht", str(m["host_name"]))
+		}},
+		{name: "service_installed", severity: "high", eventCode: 7045, score: 8.0, query: simpleEvent(7045, 1), summary: func(m map[string]any) string {
+			return fmt.Sprintf("Neuer Dienst auf %s installiert", str(m["host_name"]))
+		}},
+		{name: "account_lockout", severity: "medium", eventCode: 4740, score: 5.5, query: func(s, e time.Time, t string) string {
+			return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4740 GROUP BY host_name,user_name,workstation HAVING cnt>=1`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
+		}, summary: func(m map[string]any) string {
+			return fmt.Sprintf("Account-Lockout: %s; Caller %s; DC/Host %s (%d×)", str(m["user_name"]), fallback(str(m["workstation"]), "unbekannt"), str(m["host_name"]), int64(num(m["cnt"])))
+		}},
+		{name: "failed_logon_burst", severity: "high", eventCode: 4625, score: 7.5, query: func(s, e time.Time, t string) string {
+			return fmt.Sprintf(`SELECT host_name, target_user AS user_name, source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND target_user!='' GROUP BY host_name,user_name,source_ip,workstation HAVING cnt>=20`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
+		}, summary: func(m map[string]any) string {
+			return fmt.Sprintf("%d fehlgeschlagene Logons für %s auf %s", int64(num(m["cnt"])), str(m["user_name"]), str(m["host_name"]))
+		}},
+		{name: "password_spray", severity: "high", eventCode: 4625, score: 8.5, query: func(s, e time.Time, t string) string {
+			return fmt.Sprintf(`SELECT '' AS host_name, '' AS user_name, source_ip, '' AS workstation, uniqExact(target_user) users, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=4625 AND source_ip!='' AND target_user!='' GROUP BY source_ip HAVING users>=10 AND cnt>=20`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
+		}, summary: func(m map[string]any) string {
+			return fmt.Sprintf("Password-Spray von %s gegen %.0f Benutzer (%d Versuche)", str(m["source_ip"]), num(m["users"]), int64(num(m["cnt"])))
+		}},
+		{name: "privileged_group_change", severity: "critical", eventCode: 4728, score: 9.2, query: func(s, e time.Time, t string) string {
+			return fmt.Sprintf(`SELECT host_name, target_user AS user_name, '' AS source_ip, workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code IN (4728,4732,4756) GROUP BY host_name,user_name,workstation HAVING cnt>=1`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)))
+		}, summary: func(m map[string]any) string {
+			return fmt.Sprintf("Privilegierte Gruppenmitgliedschaft geändert: %s auf %s", str(m["user_name"]), str(m["host_name"]))
+		}},
+	}
+}
+func simpleEvent(id uint32, min int) func(time.Time, time.Time, string) string {
+	return func(s, e time.Time, t string) string {
+		return fmt.Sprintf(`SELECT host_name, '' AS user_name, '' AS source_ip, '' AS workstation, uniqExact(event_uid) cnt, min(event_time) window_start, max(event_time) window_end FROM siem.events WHERE tenant_id=%s AND event_time>=%s AND event_time<%s AND event_code=%d GROUP BY host_name HAVING cnt>=%d`, clickhouse.Q(t), clickhouse.Q(ts(s)), clickhouse.Q(ts(e)), id, min)
+	}
+}
+func ts(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000") }
+func str(v any) string {
+	if v == nil {
+		return ""
+	}
+	return fmt.Sprint(v)
+}
+func num(v any) float64 {
+	switch x := v.(type) {
+	case float64:
+		return x
+	case jsonNumber:
+		return x.Float()
+	default:
+		f, _ := strconv.ParseFloat(fmt.Sprint(v), 64)
+		return f
+	}
+}
+
+type jsonNumber string
+
+func (n jsonNumber) Float() float64 { f, _ := strconv.ParseFloat(string(n), 64); return f }
+func timeVal(v any, d time.Time) time.Time {
+	s := str(v)
+	for _, layout := range []string{"2006-01-02 15:04:05.999999", "2006-01-02 15:04:05", "2006-01-02T15:04:05Z07:00"} {
+		if t, e := time.Parse(layout, s); e == nil {
+			return t.UTC()
+		}
+	}
+	return d
+}
+func fingerprint(v ...string) string {
+	h := sha256.Sum256([]byte(strings.Join(v, "|")))
+	return hex.EncodeToString(h[:])
+}
+func fallback(v, d string) string { if strings.TrimSpace(v) == "" { return d }; return v }
+func max64(a, b int64) int64 {
+	if a > b {
+		return a
+	}
+	return b
+}

internal/ingress/server.go

@@ -0,0 +1,178 @@
+package ingress
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"example.com/siem-greenfield/internal/config"
+	"example.com/siem-greenfield/internal/contracts"
+	"example.com/siem-greenfield/internal/metrics"
+	"example.com/siem-greenfield/internal/postgres"
+	"example.com/siem-greenfield/internal/queue"
+)
+
+func Run(ctx context.Context, cfg config.Config) error {
+	pg, err := postgres.Open(ctx, cfg.PostgresURL)
+	if err != nil {
+		return fmt.Errorf("postgres: %w", err)
+	}
+	defer pg.Close()
+	prod := queue.NewProducer(cfg.KafkaBrokers, cfg.KafkaTopic)
+	defer prod.Close()
+	m := metrics.New()
+	accepted := m.Counter("siem_ingress_events_accepted_total")
+	rejected := m.Counter("siem_ingress_requests_rejected_total")
+	batches := m.Counter("siem_ingress_batches_total")
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", m.Handler())
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, 200, map[string]string{"status": "ok"}) })
+	mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
+		cctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
+		defer cancel()
+		if e := pg.Pool.Ping(cctx); e != nil {
+			writeJSON(w, 503, map[string]string{"status": "not_ready"})
+			return
+		}
+		writeJSON(w, 200, map[string]string{"status": "ready"})
+	})
+	mux.HandleFunc("/ingest", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			rejected.Add(1)
+			writeJSON(w, 405, map[string]string{"error": "method not allowed"})
+			return
+		}
+		apiKey := strings.TrimSpace(r.Header.Get("X-API-Key"))
+		if apiKey == "" {
+			rejected.Add(1)
+			writeJSON(w, 401, map[string]string{"error": "missing api key"})
+			return
+		}
+		r.Body = http.MaxBytesReader(w, r.Body, cfg.MaxBodyBytes)
+		defer r.Body.Close()
+		dec := json.NewDecoder(r.Body)
+		dec.DisallowUnknownFields()
+		var batch []contracts.LogPayload
+		if e := dec.Decode(&batch); e != nil || (func() bool { var x any; return dec.Decode(&x) != io.EOF })() {
+			rejected.Add(1)
+			writeJSON(w, 400, map[string]string{"error": "invalid json"})
+			return
+		}
+		if len(batch) == 0 || len(batch) > cfg.MaxBatchEvents {
+			rejected.Add(1)
+			writeJSON(w, 400, map[string]string{"error": "invalid batch size"})
+			return
+		}
+		host := strings.TrimSpace(batch[0].Hostname)
+		for i := range batch {
+			if e := validate(&batch[i]); e != nil {
+				rejected.Add(1)
+				writeJSON(w, 400, map[string]string{"error": fmt.Sprintf("invalid payload at index %d: %v", i, e)})
+				return
+			}
+			if batch[i].Hostname != host {
+				rejected.Add(1)
+				writeJSON(w, 400, map[string]string{"error": "all events in a batch must use the same hostname"})
+				return
+			}
+		}
+		cctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
+		defer cancel()
+		agentID, e := pg.AuthenticateOrEnroll(cctx, cfg.TenantID, host, apiKey, strings.TrimSpace(r.Header.Get("X-Enrollment-Key")), cfg.EnrollmentKey, postgres.ClientIP(r.RemoteAddr))
+		if e != nil {
+			rejected.Add(1)
+			if errors.Is(e, postgres.ErrUnauthorized) {
+				writeJSON(w, 401, map[string]string{"error": "invalid api key or hostname"})
+			} else {
+				log.Printf("auth: %v", e)
+				writeJSON(w, 503, map[string]string{"error": "control plane unavailable"})
+			}
+			return
+		}
+		env := contracts.IngestEnvelope{Version: 1, TenantID: cfg.TenantID, AgentID: agentID, BatchUID: batchUID(agentID, batch), RemoteIP: postgres.ClientIP(r.RemoteAddr), ReceivedAt: time.Now().UTC(), Events: batch}
+		payload, e := json.Marshal(env)
+		if e != nil {
+			writeJSON(w, 500, map[string]string{"error": "internal error"})
+			return
+		}
+		qctx, qcancel := context.WithTimeout(r.Context(), 8*time.Second)
+		defer qcancel()
+		if e = prod.Write(qctx, agentID, payload); e != nil {
+			log.Printf("queue write: %v", e)
+			writeJSON(w, 503, map[string]string{"error": "ingest queue unavailable"})
+			return
+		}
+		accepted.Add(uint64(len(batch)))
+		batches.Add(1)
+		writeJSON(w, 202, map[string]int{"accepted": len(batch)})
+	})
+	srv := &http.Server{Addr: cfg.ServiceAddr, Handler: withLimits(mux), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, IdleTimeout: 60 * time.Second}
+	go func() {
+		<-ctx.Done()
+		cctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(cctx)
+	}()
+	log.Printf("ingress listening on %s", cfg.ServiceAddr)
+	e := srv.ListenAndServe()
+	if e == http.ErrServerClosed {
+		return nil
+	}
+	return e
+}
+func batchUID(agentID string, batch []contracts.LogPayload) string {
+	b, _ := json.Marshal(batch)
+	h := sha256.New()
+	_, _ = h.Write([]byte(agentID))
+	_, _ = h.Write([]byte{0})
+	_, _ = h.Write(b)
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+func validate(p *contracts.LogPayload) error {
+	p.Hostname = strings.TrimSpace(p.Hostname)
+	p.Channel = strings.TrimSpace(p.Channel)
+	p.Source = strings.TrimSpace(p.Source)
+	if p.Hostname == "" || len(p.Hostname) > 255 {
+		return fmt.Errorf("invalid host")
+	}
+	if p.Channel == "" || len(p.Channel) > 128 {
+		return fmt.Errorf("invalid channel")
+	}
+	if p.Source == "" || len(p.Source) > 255 {
+		return fmt.Errorf("invalid source")
+	}
+	if p.EventID == 0 {
+		return fmt.Errorf("event id required")
+	}
+	if p.Time.IsZero() {
+		return fmt.Errorf("ts required")
+	}
+	if strings.TrimSpace(p.Message) == "" && p.Metadata == nil {
+		return fmt.Errorf("either msg or meta required")
+	}
+	if len(p.Message) > 2*1024*1024 {
+		return fmt.Errorf("msg too large")
+	}
+	return nil
+}
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(v)
+}
+func withLimits(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		w.Header().Set("Cache-Control", "no-store")
+		next.ServeHTTP(w, r)
+	})
+}

internal/ingress/server_test.go

@@ -0,0 +1,29 @@
+package ingress
+
+import (
+	"testing"
+	"time"
+
+	"example.com/siem-greenfield/internal/contracts"
+)
+
+func TestValidateCompatiblePayload(t *testing.T) {
+	p := contracts.LogPayload{Hostname: "PC01", Channel: "Security", EventID: 4625, Source: "agent", Time: time.Now().UTC(), Metadata: &contracts.EventMetadataPayload{TargetUser: "alice"}}
+	if err := validate(&p); err != nil {
+		t.Fatalf("valid payload rejected: %v", err)
+	}
+}
+
+func TestValidateRequiresMessageOrMetadata(t *testing.T) {
+	p := contracts.LogPayload{Hostname: "PC01", Channel: "Security", EventID: 4625, Source: "agent", Time: time.Now().UTC()}
+	if err := validate(&p); err == nil {
+		t.Fatal("expected validation error")
+	}
+}
+
+func TestBatchUIDIsDeterministic(t *testing.T) {
+	b := []contracts.LogPayload{{Hostname: "PC01", Channel: "Security", EventID: 4625, Source: "agent", Time: time.Date(2026,7,23,12,0,0,0,time.UTC), Metadata: &contracts.EventMetadataPayload{TargetUser: "alice"}}}
+	a := batchUID("agent-1", b)
+	if a == "" || a != batchUID("agent-1", b) { t.Fatalf("batch uid is not deterministic") }
+	if a == batchUID("agent-2", b) { t.Fatalf("batch uid must be scoped to agent") }
+}

internal/metrics/metrics.go

@@ -0,0 +1,67 @@
+package metrics
+
+import (
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+)
+
+type Registry struct {
+	mu       sync.RWMutex
+	counters map[string]*atomic.Uint64
+	gauges   map[string]*atomic.Int64
+}
+
+func New() *Registry {
+	return &Registry{counters: map[string]*atomic.Uint64{}, gauges: map[string]*atomic.Int64{}}
+}
+func (r *Registry) Counter(name string) *atomic.Uint64 {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if c := r.counters[name]; c != nil {
+		return c
+	}
+	c := &atomic.Uint64{}
+	r.counters[name] = c
+	return c
+}
+func (r *Registry) Gauge(name string) *atomic.Int64 {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if g := r.gauges[name]; g != nil {
+		return g
+	}
+	g := &atomic.Int64{}
+	r.gauges[name] = g
+	return g
+}
+func (r *Registry) Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/plain; version=0.0.4")
+		r.mu.RLock()
+		defer r.mu.RUnlock()
+		var lines []string
+		for n, c := range r.counters {
+			lines = append(lines, fmt.Sprintf("%s %d", sanitize(n), c.Load()))
+		}
+		for n, g := range r.gauges {
+			lines = append(lines, fmt.Sprintf("%s %d", sanitize(n), g.Load()))
+		}
+		sort.Strings(lines)
+		fmt.Fprintln(w, strings.Join(lines, "\n"))
+	})
+}
+func sanitize(s string) string {
+	var b strings.Builder
+	for i, r := range s {
+		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9' && i > 0) || r == '_' || r == ':' {
+			b.WriteRune(r)
+		} else {
+			b.WriteByte('_')
+		}
+	}
+	return b.String()
+}

internal/normalize/normalize.go

@@ -0,0 +1,297 @@
+package normalize
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/xml"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"example.com/siem-greenfield/internal/contracts"
+)
+
+type Parsed struct {
+	Computer, Provider, TargetUser, TargetDomain, SubjectUser, SubjectDomain, Workstation, SrcIP, SrcPort, DstIP, DstPort, LogonType, ProcessName, ParentProcessName, CommandLine, AuthPackage, LogonProcess, Status, SubStatus, FailureReason string
+	Attributes                                                                                                                                                                                                                                 map[string]string
+}
+
+func Event(env contracts.IngestEnvelope, p contracts.LogPayload, rawKey string, partition int, offset int64, idx int) contracts.CanonicalEvent {
+	parsed := parseXML(p.Message)
+	if p.Metadata != nil {
+		overlay(&parsed, *p.Metadata)
+	}
+	host := first(parsed.Computer, p.Hostname)
+	provider := first(parsed.Provider, p.Source)
+	cat, action, outcome, sev := classify(p.Channel, p.EventID)
+	user := first(parsed.TargetUser, parsed.SubjectUser)
+	msg := strings.TrimSpace(p.Message)
+	if strings.HasPrefix(msg, "<") {
+		msg = ""
+	}
+	if len(msg) > 4096 {
+		msg = msg[:4096]
+	}
+	attrs := parsed.Attributes
+	if attrs == nil {
+		attrs = map[string]string{}
+	}
+	eventUID := env.BatchUID
+	if eventUID == "" {
+		eventUID = strconv.Itoa(partition) + ":" + strconv.FormatInt(offset, 10)
+	}
+	eventUID += ":" + strconv.Itoa(idx)
+	return contracts.CanonicalEvent{
+		EventUID: eventUID, QueuePartition: int32(partition), QueueOffset: offset,
+		TenantID: env.TenantID, EventTime: p.Time.UTC().Format("2006-01-02 15:04:05.000"), IngestTime: env.ReceivedAt.UTC().Format("2006-01-02 15:04:05.000"), AgentID: env.AgentID, HostName: host, SourceType: p.Source, Channel: p.Channel, Provider: provider, EventCode: p.EventID, Category: cat, Action: action, Outcome: outcome, Severity: sev,
+		UserName: cleanUser(user), UserDomain: first(parsed.TargetDomain, parsed.SubjectDomain), SubjectUser: cleanUser(parsed.SubjectUser), SubjectDomain: parsed.SubjectDomain, TargetUser: cleanUser(parsed.TargetUser), TargetDomain: parsed.TargetDomain, SourceIP: normalizeIP(parsed.SrcIP), SourcePort: parsePort(parsed.SrcPort), DestinationIP: normalizeIP(parsed.DstIP), DestinationPort: parsePort(parsed.DstPort), Workstation: parsed.Workstation, LogonType: parsed.LogonType, AuthenticationPackage: parsed.AuthPackage, LogonProcess: parsed.LogonProcess, StatusCode: parsed.Status, SubStatusCode: parsed.SubStatus, FailureReason: parsed.FailureReason, ProcessPath: parsed.ProcessName, ParentProcessPath: parsed.ParentProcessName, CommandLine: truncate(parsed.CommandLine, 8192), Message: msg, Attributes: attrs, RawObjectKey: rawKey, RawIndex: uint32(idx), PayloadHash: fingerprint(p), SchemaVersion: 1, ParserVersion: 1, IngestDelayMS: env.ReceivedAt.Sub(p.Time).Milliseconds(),
+	}
+}
+
+func overlay(p *Parsed, m contracts.EventMetadataPayload) {
+	p.Computer = first(m.Computer, p.Computer)
+	p.Provider = first(m.ProviderName, p.Provider)
+	p.TargetUser = first(m.TargetUser, p.TargetUser)
+	p.TargetDomain = first(m.TargetDomain, p.TargetDomain)
+	p.SubjectUser = first(m.SubjectUser, p.SubjectUser)
+	p.SubjectDomain = first(m.SubjectDomain, p.SubjectDomain)
+	p.Workstation = first(m.Workstation, m.Device, p.Workstation)
+	p.SrcIP = first(m.SrcIP, p.SrcIP)
+	p.SrcPort = first(m.SrcPort, p.SrcPort)
+	p.DstIP = first(m.DstIP, p.DstIP)
+	p.DstPort = first(m.DstPort, p.DstPort)
+	p.LogonType = first(m.LogonType, p.LogonType)
+	p.ProcessName = first(m.ProcessName, p.ProcessName)
+	p.ParentProcessName = first(m.ParentProcessName, p.ParentProcessName)
+	p.CommandLine = first(m.CommandLine, p.CommandLine)
+	p.AuthPackage = first(m.AuthenticationPackage, p.AuthPackage)
+	p.LogonProcess = first(m.LogonProcess, p.LogonProcess)
+	p.Status = first(m.StatusText, p.Status)
+	p.SubStatus = first(m.SubStatusText, p.SubStatus)
+	p.FailureReason = first(m.FailureReason, p.FailureReason)
+}
+
+func parseXML(s string) Parsed {
+	out := Parsed{Attributes: map[string]string{}}
+	if !strings.HasPrefix(strings.TrimSpace(s), "<") {
+		return out
+	}
+	dec := xml.NewDecoder(strings.NewReader(s))
+	var path []string
+	dataName := ""
+	for {
+		tok, e := dec.Token()
+		if e == io.EOF {
+			break
+		}
+		if e != nil {
+			return out
+		}
+		switch t := tok.(type) {
+		case xml.StartElement:
+			path = append(path, t.Name.Local)
+			if t.Name.Local == "Provider" {
+				for _, a := range t.Attr {
+					if a.Name.Local == "Name" {
+						out.Provider = strings.TrimSpace(a.Value)
+					}
+				}
+			}
+			if t.Name.Local == "Data" {
+				dataName = ""
+				for _, a := range t.Attr {
+					if a.Name.Local == "Name" {
+						dataName = strings.TrimSpace(a.Value)
+					}
+				}
+			}
+		case xml.EndElement:
+			if len(path) > 0 {
+				path = path[:len(path)-1]
+			}
+			if t.Name.Local == "Data" {
+				dataName = ""
+			}
+		case xml.CharData:
+			v := strings.TrimSpace(string(t))
+			if v == "" {
+				continue
+			}
+			if ends(path, "System", "Computer") {
+				out.Computer = v
+				continue
+			}
+			if dataName != "" {
+				if !promotedField(dataName) && len(out.Attributes) < 32 {
+					out.Attributes[dataName] = truncate(v, 1024)
+				}
+				switch dataName {
+				case "TargetUserName":
+					out.TargetUser = v
+				case "TargetDomainName":
+					out.TargetDomain = v
+				case "SubjectUserName":
+					out.SubjectUser = v
+				case "SubjectDomainName":
+					out.SubjectDomain = v
+				case "WorkstationName", "CallerComputerName":
+					out.Workstation = v
+				case "IpAddress", "SourceAddress":
+					out.SrcIP = v
+				case "IpPort", "SourcePort":
+					out.SrcPort = v
+				case "DestinationAddress", "DestAddress":
+					out.DstIP = v
+				case "DestinationPort", "DestPort":
+					out.DstPort = v
+				case "LogonType":
+					out.LogonType = v
+				case "ProcessName", "NewProcessName":
+					out.ProcessName = v
+				case "ParentProcessName":
+					out.ParentProcessName = v
+				case "CommandLine", "ProcessCommandLine":
+					out.CommandLine = v
+				case "AuthenticationPackageName":
+					out.AuthPackage = v
+				case "LogonProcessName":
+					out.LogonProcess = v
+				case "Status":
+					out.Status = v
+				case "SubStatus":
+					out.SubStatus = v
+				case "FailureReason":
+					out.FailureReason = v
+				}
+			}
+		}
+	}
+	return out
+}
+func promotedField(name string) bool {
+	switch name {
+	case "TargetUserName", "TargetDomainName", "SubjectUserName", "SubjectDomainName",
+		"WorkstationName", "CallerComputerName", "IpAddress", "SourceAddress", "IpPort", "SourcePort",
+		"DestinationAddress", "DestAddress", "DestinationPort", "DestPort", "LogonType",
+		"ProcessName", "NewProcessName", "ParentProcessName", "CommandLine", "ProcessCommandLine",
+		"AuthenticationPackageName", "LogonProcessName", "Status", "SubStatus", "FailureReason":
+		return true
+	default:
+		return false
+	}
+}
+
+func ends(path []string, p ...string) bool {
+	if len(path) < len(p) {
+		return false
+	}
+	o := len(path) - len(p)
+	for i := range p {
+		if path[o+i] != p[i] {
+			return false
+		}
+	}
+	return true
+}
+func first(v ...string) string {
+	for _, s := range v {
+		if s = strings.TrimSpace(s); s != "" && s != "-" {
+			return s
+		}
+	}
+	return ""
+}
+func cleanUser(v string) string {
+	v = strings.TrimSpace(v)
+	if v == "-" {
+		return ""
+	}
+	return v
+}
+func normalizeIP(v string) string {
+	v = strings.TrimSpace(v)
+	if v == "" || v == "-" || v == "::1" || v == "127.0.0.1" {
+		return ""
+	}
+	ip := net.ParseIP(v)
+	if ip == nil {
+		return ""
+	}
+	if x := ip.To4(); x != nil {
+		return x.String()
+	}
+	return ip.String()
+}
+func parsePort(v string) uint16 {
+	n, e := strconv.ParseUint(strings.TrimSpace(v), 10, 16)
+	if e != nil {
+		return 0
+	}
+	return uint16(n)
+}
+func truncate(v string, n int) string {
+	if len(v) > n {
+		return v[:n]
+	}
+	return v
+}
+func fingerprint(p contracts.LogPayload) string {
+	b, _ := json.Marshal(p)
+	h := sha256.Sum256(b)
+	return hex.EncodeToString(h[:])
+}
+
+func classify(channel string, id uint32) (string, string, string, uint8) {
+	if strings.EqualFold(channel, "Security") {
+		switch id {
+		case 4624:
+			return "authentication", "logon", "success", 2
+		case 4625:
+			return "authentication", "logon", "failure", 3
+		case 4648:
+			return "authentication", "explicit_credentials", "unknown", 3
+		case 4672:
+			return "iam", "special_privileges", "success", 3
+		case 4720:
+			return "iam", "user_created", "success", 4
+		case 4726:
+			return "iam", "user_deleted", "success", 4
+		case 4728, 4732, 4756:
+			return "iam", "privileged_group_membership_changed", "success", 5
+		case 4740:
+			return "authentication", "account_locked", "failure", 4
+		case 4768:
+			return "authentication", "kerberos_tgt", "unknown", 2
+		case 4769:
+			return "authentication", "kerberos_service_ticket", "unknown", 2
+		case 4771:
+			return "authentication", "kerberos_preauth", "failure", 3
+		case 4776:
+			return "authentication", "credential_validation", "unknown", 2
+		case 1102:
+			return "audit", "audit_log_cleared", "success", 5
+		}
+	}
+	switch id {
+	case 7045:
+		return "persistence", "service_installed", "success", 5
+	case 1074:
+		return "host", "shutdown_requested", "success", 2
+	case 6005:
+		return "host", "eventlog_started", "success", 1
+	case 6006:
+		return "host", "eventlog_stopped", "success", 2
+	}
+	return "event", "observed", "unknown", 1
+}
+
+func ArchiveKey(t time.Time, agent, batchUID string, partition int, offset int64) string {
+	id := strings.TrimSpace(batchUID)
+	if id == "" {
+		id = "p" + strconv.Itoa(partition) + "-o" + strconv.FormatInt(offset, 10)
+	}
+	return t.UTC().Format("2006/01/02/15") + "/" + strings.ReplaceAll(agent, "/", "_") + "-" + id + ".json.gz"
+}

internal/normalize/normalize_test.go

@@ -0,0 +1,37 @@
+package normalize
+
+import (
+	"testing"
+	"time"
+
+	"example.com/siem-greenfield/internal/contracts"
+)
+
+func TestEvent4740ExtractsCallerComputer(t *testing.T) {
+	xml := `<Event><System><Provider Name="Microsoft-Windows-Security-Auditing"/><Computer>DC01.example.local</Computer></System><EventData><Data Name="TargetUserName">alice</Data><Data Name="CallerComputerName">CLIENT-42</Data></EventData></Event>`
+	env := contracts.IngestEnvelope{TenantID: "default", AgentID: "agent-1", BatchUID: "batch-abc", ReceivedAt: time.Date(2026, 7, 23, 12, 0, 1, 0, time.UTC)}
+	p := contracts.LogPayload{Hostname: "DC01", Channel: "Security", EventID: 4740, Source: "windows-agent", Time: time.Date(2026, 7, 23, 12, 0, 0, 0, time.UTC), Message: xml}
+	e := Event(env, p, "2026/07/raw.json.gz", 3, 99, 4)
+	if e.EventUID != "batch-abc:4" {
+		t.Fatalf("event uid = %q", e.EventUID)
+	}
+	if e.TargetUser != "alice" || e.Workstation != "CLIENT-42" {
+		t.Fatalf("unexpected normalized event: %#v", e)
+	}
+	if e.Action != "account_locked" || e.Severity != 4 {
+		t.Fatalf("unexpected classification: action=%s severity=%d", e.Action, e.Severity)
+	}
+	if e.Message != "" {
+		t.Fatalf("raw XML must not be stored as message")
+	}
+	if len(e.Attributes) != 0 {
+		t.Fatalf("promoted fields must not be duplicated in attributes: %#v", e.Attributes)
+	}
+}
+
+func TestWMI5857IsGenericNotDetectionClass(t *testing.T) {
+	cat, action, _, sev := classify("Microsoft-Windows-WMI-Activity/Operational", 5857)
+	if cat != "event" || action != "observed" || sev != 1 {
+		t.Fatalf("unexpected WMI classification: %s %s %d", cat, action, sev)
+	}
+}

internal/postgres/store.go

@@ -0,0 +1,184 @@
+package postgres
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+var ErrUnauthorized = errors.New("unauthorized")
+
+type Store struct{ Pool *pgxpool.Pool }
+
+type Agent struct {
+	ID        string    `json:"id"`
+	Hostname  string    `json:"hostname"`
+	LastIP    string    `json:"last_ip"`
+	Enabled   bool      `json:"enabled"`
+	FirstSeen time.Time `json:"first_seen"`
+	LastSeen  time.Time `json:"last_seen"`
+}
+type Detection struct {
+	ID          int64     `json:"id"`
+	Fingerprint string    `json:"fingerprint"`
+	RuleName    string    `json:"rule_name"`
+	Severity    string    `json:"severity"`
+	Status      string    `json:"status"`
+	Hostname    string    `json:"hostname"`
+	UserName    string    `json:"user_name"`
+	SourceIP    string    `json:"source_ip"`
+	Workstation string    `json:"workstation"`
+	Summary     string    `json:"summary"`
+	EventCode   uint32    `json:"event_code"`
+	Score       float64   `json:"score"`
+	WindowStart time.Time `json:"window_start"`
+	WindowEnd   time.Time `json:"window_end"`
+	FirstSeen   time.Time `json:"first_seen"`
+	LastSeen    time.Time `json:"last_seen"`
+	Count       int64     `json:"count"`
+}
+
+func Open(ctx context.Context, url string) (*Store, error) {
+	p, e := pgxpool.New(ctx, url)
+	if e != nil {
+		return nil, e
+	}
+	cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	if e = p.Ping(cctx); e != nil {
+		p.Close()
+		return nil, e
+	}
+	return &Store{Pool: p}, nil
+}
+func (s *Store) Close() { s.Pool.Close() }
+
+func (s *Store) AuthenticateOrEnroll(ctx context.Context, tenant, hostname, apiKey, enrollmentKey, expectedEnrollment, remoteIP string) (string, error) {
+	hostname = strings.TrimSpace(hostname)
+	apiKey = strings.TrimSpace(apiKey)
+	var id, hash string
+	var enabled bool
+	err := s.Pool.QueryRow(ctx, `SELECT id::text, api_key_hash, enabled FROM agents WHERE tenant_id=$1 AND hostname=$2`, tenant, hostname).Scan(&id, &hash, &enabled)
+	if err != nil {
+		if !errors.Is(err, pgx.ErrNoRows) {
+			return "", err
+		}
+		if enrollmentKey == "" || !secureEqual(hashHex(enrollmentKey), hashHex(expectedEnrollment)) {
+			return "", ErrUnauthorized
+		}
+		err = s.Pool.QueryRow(ctx, `INSERT INTO agents(tenant_id,hostname,api_key_hash,last_ip) VALUES($1,$2,$3,$4) ON CONFLICT(tenant_id,hostname) DO NOTHING RETURNING id::text`, tenant, hostname, hashHex(apiKey), remoteIP).Scan(&id)
+		if err == nil {
+			return id, nil
+		}
+		if !errors.Is(err, pgx.ErrNoRows) {
+			return "", err
+		}
+		// A concurrent first request may have enrolled the host between SELECT and INSERT.
+		err = s.Pool.QueryRow(ctx, `SELECT id::text, api_key_hash, enabled FROM agents WHERE tenant_id=$1 AND hostname=$2`, tenant, hostname).Scan(&id, &hash, &enabled)
+		if err != nil || !enabled || !secureEqual(strings.ToLower(hash), hashHex(apiKey)) {
+			return "", ErrUnauthorized
+		}
+		return id, nil
+	}
+	if !enabled || !secureEqual(strings.ToLower(hash), hashHex(apiKey)) {
+		return "", ErrUnauthorized
+	}
+	_, err = s.Pool.Exec(ctx, `UPDATE agents SET last_seen=now(), last_ip=$3 WHERE tenant_id=$1 AND hostname=$2`, tenant, hostname, remoteIP)
+	return id, err
+}
+
+func (s *Store) ListAgents(ctx context.Context, tenant string) ([]Agent, error) {
+	rows, e := s.Pool.Query(ctx, `SELECT id::text,hostname,enabled,COALESCE(last_ip,''),first_seen,last_seen FROM agents WHERE tenant_id=$1 ORDER BY last_seen DESC`, tenant)
+	if e != nil {
+		return nil, e
+	}
+	defer rows.Close()
+	var out []Agent
+	for rows.Next() {
+		var a Agent
+		if e := rows.Scan(&a.ID, &a.Hostname, &a.Enabled, &a.LastIP, &a.FirstSeen, &a.LastSeen); e != nil {
+			return nil, e
+		}
+		out = append(out, a)
+	}
+	return out, rows.Err()
+}
+
+func (s *Store) UpsertDetection(ctx context.Context, d Detection, tenant string) error {
+	_, e := s.Pool.Exec(ctx, `INSERT INTO detections(tenant_id,fingerprint,rule_name,severity,status,hostname,user_name,source_ip,workstation,event_code,score,window_start,window_end,summary,hit_count,first_seen,last_seen)
+VALUES($1,$2,$3,$4,'open',$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$11,$12)
+ON CONFLICT(tenant_id,fingerprint) DO UPDATE SET
+  first_seen=LEAST(detections.first_seen,EXCLUDED.first_seen),
+  last_seen=GREATEST(detections.last_seen,EXCLUDED.last_seen),
+  window_start=LEAST(detections.window_start,EXCLUDED.window_start),
+  window_end=GREATEST(detections.window_end,EXCLUDED.window_end),
+  hit_count=GREATEST(detections.hit_count,EXCLUDED.hit_count),
+  score=GREATEST(detections.score,EXCLUDED.score),
+  summary=EXCLUDED.summary, workstation=EXCLUDED.workstation, updated_at=now()`, tenant, d.Fingerprint, d.RuleName, d.Severity, d.Hostname, d.UserName, d.SourceIP, d.Workstation, d.EventCode, d.Score, d.WindowStart, d.WindowEnd, d.Summary, d.Count)
+	return e
+}
+
+func (s *Store) ListDetections(ctx context.Context, tenant string, limit int) ([]Detection, error) {
+	rows, e := s.Pool.Query(ctx, `SELECT id,fingerprint,rule_name,severity,status,hostname,user_name,source_ip,workstation,event_code,score,window_start,window_end,summary,hit_count,first_seen,last_seen FROM detections WHERE tenant_id=$1 ORDER BY last_seen DESC LIMIT $2`, tenant, limit)
+	if e != nil {
+		return nil, e
+	}
+	defer rows.Close()
+	var out []Detection
+	for rows.Next() {
+		var d Detection
+		if e := rows.Scan(&d.ID, &d.Fingerprint, &d.RuleName, &d.Severity, &d.Status, &d.Hostname, &d.UserName, &d.SourceIP, &d.Workstation, &d.EventCode, &d.Score, &d.WindowStart, &d.WindowEnd, &d.Summary, &d.Count, &d.FirstSeen, &d.LastSeen); e != nil {
+			return nil, e
+		}
+		out = append(out, d)
+	}
+	return out, rows.Err()
+}
+func (s *Store) UpdateDetectionStatus(ctx context.Context, tenant string, id int64, status string) error {
+	if status != "open" && status != "investigating" && status != "closed" && status != "false_positive" {
+		return fmt.Errorf("invalid status")
+	}
+	_, e := s.Pool.Exec(ctx, `UPDATE detections SET status=$3, updated_at=now() WHERE tenant_id=$1 AND id=$2`, tenant, id, status)
+	return e
+}
+func (s *Store) DetectionCounts(ctx context.Context, tenant string) (map[string]int64, error) {
+	rows, e := s.Pool.Query(ctx, `SELECT severity,count(*) FROM detections WHERE tenant_id=$1 AND status IN ('open','investigating') GROUP BY severity`, tenant)
+	if e != nil {
+		return nil, e
+	}
+	defer rows.Close()
+	out := map[string]int64{}
+	for rows.Next() {
+		var k string
+		var n int64
+		if e := rows.Scan(&k, &n); e != nil {
+			return nil, e
+		}
+		out[k] = n
+	}
+	return out, rows.Err()
+}
+
+func hashHex(v string) string { h := sha256.Sum256([]byte(v)); return hex.EncodeToString(h[:]) }
+func secureEqual(a, b string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
+}
+func ClientIP(remote string) string {
+	h, _, e := net.SplitHostPort(remote)
+	if e == nil {
+		return h
+	}
+	return remote
+}

internal/processor/processor.go

@@ -0,0 +1,88 @@
+package processor
+
+import (
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	"example.com/siem-greenfield/internal/clickhouse"
+	"example.com/siem-greenfield/internal/config"
+	"example.com/siem-greenfield/internal/contracts"
+	"example.com/siem-greenfield/internal/normalize"
+	"example.com/siem-greenfield/internal/queue"
+)
+
+func Run(ctx context.Context, cfg config.Config) error {
+	ch := clickhouse.New(cfg)
+	con := queue.NewConsumer(cfg.KafkaBrokers, cfg.KafkaTopic, cfg.KafkaGroup)
+	defer con.Close()
+	log.Printf("processor started topic=%s group=%s", cfg.KafkaTopic, cfg.KafkaGroup)
+	for {
+		m, e := con.Fetch(ctx)
+		if e != nil {
+			if ctx.Err() != nil {
+				return nil
+			}
+			return e
+		}
+		var env contracts.IngestEnvelope
+		if e = json.Unmarshal(m.Value, &env); e != nil {
+			log.Printf("dropping malformed queue message offset=%d: %v", m.Offset, e)
+			_ = con.Commit(ctx, m)
+			continue
+		}
+		rawKey := ""
+		if cfg.RawArchiveEnabled {
+			rawKey = normalize.ArchiveKey(env.ReceivedAt, env.AgentID, env.BatchUID, m.Partition, m.Offset)
+			if e = spool(cfg, rawKey, m.Value); e != nil {
+				log.Printf("raw spool failed offset=%d (message will be retried): %v", m.Offset, e)
+				time.Sleep(time.Second)
+				continue
+			}
+		}
+		events := make([]contracts.CanonicalEvent, 0, len(env.Events))
+		for i, p := range env.Events {
+			events = append(events, normalize.Event(env, p, rawKey, m.Partition, m.Offset, i))
+		}
+		cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+		e = ch.InsertEvents(cctx, events)
+		cancel()
+		if e != nil {
+			log.Printf("clickhouse insert failed offset=%d: %v", m.Offset, e)
+			time.Sleep(time.Second)
+			continue
+		}
+		if e = con.Commit(ctx, m); e != nil {
+			return fmt.Errorf("commit queue offset: %w", e)
+		}
+	}
+}
+func spool(cfg config.Config, key string, payload []byte) error {
+	path := filepath.Join(cfg.RawSpoolDir, filepath.FromSlash(key))
+	if e := os.MkdirAll(filepath.Dir(path), 0750); e != nil {
+		return e
+	}
+	tmp := path + ".tmp"
+	f, e := os.Create(tmp)
+	if e != nil {
+		return e
+	}
+	gz := gzip.NewWriter(f)
+	_, e = gz.Write(payload)
+	if ce := gz.Close(); e == nil {
+		e = ce
+	}
+	if ce := f.Close(); e == nil {
+		e = ce
+	}
+	if e != nil {
+		_ = os.Remove(tmp)
+		return e
+	}
+	return os.Rename(tmp, path)
+}

internal/queue/queue.go

@@ -0,0 +1,29 @@
+package queue
+
+import (
+	"context"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+)
+
+type Producer struct{ w *kafka.Writer }
+
+func NewProducer(brokers []string, topic string) *Producer {
+	return &Producer{w: &kafka.Writer{Addr: kafka.TCP(brokers...), Topic: topic, Balancer: &kafka.Hash{}, BatchSize: 100, BatchTimeout: 10 * time.Millisecond, RequiredAcks: kafka.RequireAll, Async: false}}
+}
+func (p *Producer) Write(ctx context.Context, key string, value []byte) error {
+	return p.w.WriteMessages(ctx, kafka.Message{Key: []byte(key), Value: value, Time: time.Now().UTC()})
+}
+func (p *Producer) Close() error { return p.w.Close() }
+
+type Consumer struct{ r *kafka.Reader }
+
+func NewConsumer(brokers []string, topic, group string) *Consumer {
+	return &Consumer{r: kafka.NewReader(kafka.ReaderConfig{Brokers: brokers, Topic: topic, GroupID: group, MinBytes: 1e3, MaxBytes: 16e6, MaxWait: 500 * time.Millisecond, CommitInterval: 0})}
+}
+func (c *Consumer) Fetch(ctx context.Context) (kafka.Message, error) { return c.r.FetchMessage(ctx) }
+func (c *Consumer) Commit(ctx context.Context, m kafka.Message) error {
+	return c.r.CommitMessages(ctx, m)
+}
+func (c *Consumer) Close() error { return c.r.Close() }

internal/stress/stress.go

@@ -0,0 +1,132 @@
+package stress
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"math/rand/v2"
+	"net/http"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"example.com/siem-greenfield/internal/config"
+	"example.com/siem-greenfield/internal/contracts"
+)
+
+func Run(ctx context.Context, _ config.Config, args []string) error {
+	fs := flag.NewFlagSet("stress-agent", flag.ContinueOnError)
+	url := fs.String("url", "http://127.0.0.1:8090/ingest", "")
+	api := fs.String("api-key", "stress-agent-key", "")
+	enroll := fs.String("enrollment-key", "", "")
+	host := fs.String("host", "SIEM-STRESS-01", "")
+	rate := fs.Int("rate", 1000, "events per second")
+	batch := fs.Int("batch", 100, "events per request")
+	workers := fs.Int("workers", 8, "max concurrent requests")
+	duration := fs.Duration("duration", time.Minute, "")
+	maxEvents := fs.Int64("max-events", 5_000_000, "hard event limit")
+	scenario := fs.String("scenario", "mixed", "")
+	confirm := fs.Bool("confirm-load-test", false, "")
+	if e := fs.Parse(args); e != nil {
+		return e
+	}
+	if !*confirm {
+		return fmt.Errorf("refusing to start without --confirm-load-test")
+	}
+	if *rate < 1 || *rate > 100000 || *batch < 1 || *batch > 1000 || *workers < 1 || *workers > 64 || *duration <= 0 || *duration > time.Hour || *maxEvents < 1 || *maxEvents > 50_000_000 {
+		return fmt.Errorf("unsafe load-test parameters")
+	}
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	end := time.Now().Add(*duration)
+	var sent, failed, scheduled atomic.Uint64
+	sem := make(chan struct{}, *workers)
+	var wg sync.WaitGroup
+
+	for time.Now().Before(end) && int64(scheduled.Load()) < *maxEvents {
+		select {
+		case <-ctx.Done():
+			wg.Wait()
+			return nil
+		default:
+		}
+		remaining := *maxEvents - int64(scheduled.Load())
+		n := min(*batch, *rate)
+		if int64(n) > remaining {
+			n = int(remaining)
+		}
+		if n <= 0 {
+			break
+		}
+		events := make([]contracts.LogPayload, 0, n)
+		for i := 0; i < n; i++ {
+			events = append(events, makeEvent(*host, *scenario))
+		}
+		scheduled.Add(uint64(n))
+		sem <- struct{}{}
+		wg.Add(1)
+		go func(b []contracts.LogPayload) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			data, _ := json.Marshal(b)
+			req, _ := http.NewRequestWithContext(ctx, http.MethodPost, *url, bytes.NewReader(data))
+			req.Header.Set("Content-Type", "application/json")
+			req.Header.Set("X-API-Key", *api)
+			if *enroll != "" {
+				req.Header.Set("X-Enrollment-Key", *enroll)
+			}
+			resp, e := client.Do(req)
+			if e != nil {
+				failed.Add(uint64(len(b)))
+				return
+			}
+			resp.Body.Close()
+			if resp.StatusCode/100 != 2 {
+				failed.Add(uint64(len(b)))
+				return
+			}
+			sent.Add(uint64(len(b)))
+		}(events)
+		// Pace by event count, not requests. This remains correct when rate < batch.
+		time.Sleep(time.Duration(float64(time.Second) * float64(n) / float64(*rate)))
+	}
+	wg.Wait()
+	fmt.Printf("stress test finished: accepted=%d failed=%d scheduled=%d duration=%s\n", sent.Load(), failed.Load(), scheduled.Load(), *duration)
+	return nil
+}
+
+func makeEvent(host, scenario string) contracts.LogPayload {
+	id := uint32(4624)
+	switch scenario {
+	case "failed-logon":
+		id = 4625
+	case "lockout":
+		id = 4740
+	case "catalog":
+		id = uint32(10000 + rand.IntN(30000))
+	case "mixed":
+		ids := []uint32{4624, 4625, 4740, 4768, 4769, 7045, 5857}
+		id = ids[rand.IntN(len(ids))]
+	}
+	u := "user" + strconv.Itoa(rand.IntN(500))
+	ip := fmt.Sprintf("10.%d.%d.%d", rand.IntN(250)+1, rand.IntN(250)+1, rand.IntN(250)+1)
+	return contracts.LogPayload{Hostname: host, Channel: func() string {
+		if id == 7045 {
+			return "System"
+		}
+		if id == 5857 {
+			return "Microsoft-Windows-WMI-Activity/Operational"
+		}
+		return "Security"
+	}(), EventID: id, Source: "SIEM-Stress-Agent", Time: time.Now().UTC(), Metadata: &contracts.EventMetadataPayload{ProviderName: "SIEM-Stress-Agent", TargetUser: u, Workstation: "STRESS-CLIENT-" + strconv.Itoa(rand.IntN(50)), SrcIP: ip, LogonType: "3"}}
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}