Update mit Stresstest

2026-07-18 19:46:02 +02:00
parent 10541fb22d
commit f9662780f7
4 changed files with 298 additions and 25 deletions
@@ -4614,3 +4614,73 @@ Besonders wichtig für die aktuelle `runOffHoursLoginRule`:
 SYSTEM, LOCAL SERVICE, NETWORK SERVICE, ANONYMOUS LOGON und Maschinenkonten sollten ignoriert werden.
 Zusätzlich sollte die Regel auf LogonType 2, 7, 10 und 11 begrenzt werden.
 ```
+
+---
+
+## Metadata-first Ergänzung: neue Event-IDs realistisch bewerten
+
+Eine bislang unbekannte Event-ID ist zunächst eine Änderung des Event-Inventars und noch kein Sicherheitsvorfall. Die Regel `new_event_id` besitzt deshalb nun drei Betriebsarten:
+
+- `NEW_EVENT_ID_MODE=off`: Regel vollständig deaktiviert. `event_catalog` wird weiterhin gepflegt.
+- `NEW_EVENT_ID_MODE=inventory` (Standard): bestätigte neue IDs werden höchstens als `info` mit Status `plausible` abgelegt.
+- `NEW_EVENT_ID_MODE=alert`: neue IDs in den konfigurierten Alert-Channels werden als `low` beziehungsweise bei hoher Wiederholung als `medium` geöffnet. Erst im Modus `alert` werden explizit konfigurierte High-Risk-IDs zu `medium`/`high` hochgestuft.
+
+Zusätzliche Schutzmechanismen:
+
+```env
+NEW_EVENT_ID_LEARNING_PERIOD=24h
+NEW_EVENT_ID_CONFIRM_WINDOW=15m
+NEW_EVENT_ID_MIN_COUNT=3
+NEW_EVENT_ID_IGNORE_CHANNELS=Microsoft-Windows-WMI-Activity/Operational
+NEW_EVENT_ID_ALERT_CHANNELS=Security,System
+NEW_EVENT_ID_HIGH_RISK_IDS=1102,4697,4719,7045
+```
+
+Damit erzeugt Event-ID `5857` im Channel `Microsoft-Windows-WMI-Activity/Operational` standardmäßig keine Detection mehr. Neue Agenten dürfen zunächst eine Baseline lernen. Ein einmaliges Event wird ebenfalls nicht gemeldet. Die stabile Detection-Zeitspanne basiert auf `event_catalog.first_seen`, sodass die Regel bei wiederholten Läufen keine neuen Duplikate erzeugt.
+
+Für bestehende Installationen kann anschließend ausgeführt werden:
+
+```bash
+mariadb -u root -p eventcollector < deploy/mariadb/migrations/003-new-event-id-classification.sql
+```
+
+Die Migration stuft bereits offene `new_event_id`-Meldungen aus dem WMI-Activity/Operational-Channel als `info` und `legitimate` ein.
+
+## Begrenzter Stress-Test-Agent
+
+Der Agent erzeugt ausschließlich klar markierte Metadaten (`source` und `provider` sind `SIEM-Stress-Agent`). Er besitzt harte Grenzen für Rate, Laufzeit, Worker und Gesamtzahl und startet nur mit expliziter Bestätigung.
+
+Direkt mit Go:
+
+```bash
+go run ./cmd/siem-stress-agent \
+  --url http://127.0.0.1:8090/ingest \
+  --host SIEM-STRESS-01 \
+  --api-key 'EIN-EIGENER-TEST-API-KEY' \
+  --enrollment-key 'DER-ENROLLMENT-KEY' \
+  --scenario mixed \
+  --rate 500 \
+  --batch 100 \
+  --workers 4 \
+  --duration 2m \
+  --max-events 60000 \
+  --confirm-load-test
+```
+
+Über das opt-in Compose-Profil:
+
+```bash
+cp .env.example .env
+# Kennwörter, ENROLLMENT_KEY und SIEM_STRESS_API_KEY setzen
+docker compose --profile stress run --rm stress-agent
+```
+
+Szenarien:
+
+- `normal`: erfolgreiche Logons, um den normalen Ingest-Durchsatz zu messen.
+- `failed-logon`: Event 4625 mit wechselnden Benutzern und Test-Netz-IP-Adressen.
+- `lockout`: Event 4740 mit Testbenutzern und Test-Workstations.
+- `mixed`: Mischung aus normalen Logons, Fehlern, Lockouts, Reboots und WMI 5857.
+- `catalog`: bis zu 5.000 künstliche Event-IDs im eigenen Channel `SIEM-Stress-Agent/Operational`; geeignet zum Test des Event-Katalogs, nicht für einen normalen Lasttest.
+
+Der Agent gibt akzeptierte/fehlgeschlagene Events, tatsächlichen Durchsatz, HTTP-Status sowie p50/p95/p99-Latenzen aus. Er sollte nur gegen ein System ausgeführt werden, für das eine ausdrückliche Lasttest-Freigabe besteht.
@@ -62,6 +62,13 @@ services:
      NEW_SOURCE_IP_LOOKBACK: ${NEW_SOURCE_IP_LOOKBACK}
      NEW_SOURCE_IP_WINDOW: ${NEW_SOURCE_IP_WINDOW}
      DETECTIONS_LIMIT: ${DETECTIONS_LIMIT}
+      NEW_EVENT_ID_MODE: ${NEW_EVENT_ID_MODE}
+      NEW_EVENT_ID_LEARNING_PERIOD: ${NEW_EVENT_ID_LEARNING_PERIOD}
+      NEW_EVENT_ID_CONFIRM_WINDOW: ${NEW_EVENT_ID_CONFIRM_WINDOW}
+      NEW_EVENT_ID_MIN_COUNT: ${NEW_EVENT_ID_MIN_COUNT}
+      NEW_EVENT_ID_IGNORE_CHANNELS: ${NEW_EVENT_ID_IGNORE_CHANNELS}
+      NEW_EVENT_ID_ALERT_CHANNELS: ${NEW_EVENT_ID_ALERT_CHANNELS}
+      NEW_EVENT_ID_HIGH_RISK_IDS: ${NEW_EVENT_ID_HIGH_RISK_IDS}
      STORE_RAW_XML: ${STORE_RAW_XML}
      METADATA_BUCKET: ${METADATA_BUCKET}
      EVENT_RETENTION: ${EVENT_RETENTION}
@@ -89,6 +96,32 @@ services:
      start_period: 20s
    networks:
      - dockge_default
+  stress-agent:
+    profiles: ["stress"]
+    build:
+      context: .
+      dockerfile: Dockerfile.stress
+    container_name: siem-stress-agent
+    restart: "no"
+    environment:
+      SIEM_STRESS_URL: ${SIEM_STRESS_URL}
+      SIEM_STRESS_HOST: ${SIEM_STRESS_HOST}
+      SIEM_STRESS_API_KEY: ${SIEM_STRESS_API_KEY}
+      SIEM_STRESS_ENROLLMENT_KEY: ${ENROLLMENT_KEY}
+      SIEM_STRESS_SCENARIO: ${SIEM_STRESS_SCENARIO}
+      SIEM_STRESS_RATE: ${SIEM_STRESS_RATE}
+      SIEM_STRESS_BATCH: ${SIEM_STRESS_BATCH}
+      SIEM_STRESS_WORKERS: ${SIEM_STRESS_WORKERS}
+      SIEM_STRESS_DURATION: ${SIEM_STRESS_DURATION}
+      SIEM_STRESS_MAX_EVENTS: ${SIEM_STRESS_MAX_EVENTS}
+      SIEM_STRESS_TIMEOUT: ${SIEM_STRESS_TIMEOUT}
+      SIEM_STRESS_CONFIRM: ${SIEM_STRESS_CONFIRM}
+    depends_on:
+      siem-backend:
+        condition: service_healthy
+    networks:
+      - dockge_default
+
  prometheus:
    image: prom/prometheus:latest
    container_name: siem-prometheus
@@ -67,3 +67,25 @@ PARTITION_AHEAD=24h
 PARTITION_BEHIND=6h
 # Compatibility fallback; table-specific retention values above take precedence.
 PARTITION_RETENTION=720h
+
+# New Event-ID classification
+NEW_EVENT_ID_MODE=inventory
+NEW_EVENT_ID_LEARNING_PERIOD=24h
+NEW_EVENT_ID_CONFIRM_WINDOW=15m
+NEW_EVENT_ID_MIN_COUNT=3
+NEW_EVENT_ID_IGNORE_CHANNELS=Microsoft-Windows-WMI-Activity/Operational
+NEW_EVENT_ID_ALERT_CHANNELS=Security,System
+NEW_EVENT_ID_HIGH_RISK_IDS=1102,4697,4719,7045
+
+# Optional stress-agent profile
+SIEM_STRESS_URL=http://siem-backend:8080/ingest
+SIEM_STRESS_HOST=SIEM-STRESS-01
+SIEM_STRESS_API_KEY=CHANGE-ME-STRESS-API-KEY
+SIEM_STRESS_SCENARIO=mixed
+SIEM_STRESS_RATE=200
+SIEM_STRESS_BATCH=100
+SIEM_STRESS_WORKERS=4
+SIEM_STRESS_DURATION=30s
+SIEM_STRESS_MAX_EVENTS=100000
+SIEM_STRESS_TIMEOUT=20s
+SIEM_STRESS_CONFIRM=true
@@ -1149,6 +1149,17 @@ type Config struct {
 	NewSourceIPWindow         time.Duration
 	DetectionsLimit           int

+	// A previously unseen Event ID is primarily inventory information, not an
+	// incident. The rule can be disabled, run in inventory mode, or explicitly
+	// promoted to alert mode for selected channels/Event IDs.
+	NewEventIDMode           string
+	NewEventIDLearningPeriod time.Duration
+	NewEventIDConfirmWindow  time.Duration
+	NewEventIDMinCount       int
+	NewEventIDIgnoreChannels []string
+	NewEventIDAlertChannels  []string
+	NewEventIDHighRiskIDs    map[uint32]struct{}
+
 	EnrollmentKey string

 	BaselineEnabled      bool
@@ -3965,6 +3976,14 @@ func loadConfig() Config {
 		NewSourceIPWindow:         getenvDuration("NEW_SOURCE_IP_WINDOW", 10*time.Minute),
 		DetectionsLimit:           getenvInt("DETECTIONS_LIMIT", 100),

+		NewEventIDMode:           strings.ToLower(getenv("NEW_EVENT_ID_MODE", "inventory")),
+		NewEventIDLearningPeriod: getenvDuration("NEW_EVENT_ID_LEARNING_PERIOD", 24*time.Hour),
+		NewEventIDConfirmWindow:  getenvDuration("NEW_EVENT_ID_CONFIRM_WINDOW", 15*time.Minute),
+		NewEventIDMinCount:       getenvInt("NEW_EVENT_ID_MIN_COUNT", 3),
+		NewEventIDIgnoreChannels: getenvCSV("NEW_EVENT_ID_IGNORE_CHANNELS", "Microsoft-Windows-WMI-Activity/Operational"),
+		NewEventIDAlertChannels:  getenvCSV("NEW_EVENT_ID_ALERT_CHANNELS", "Security,System"),
+		NewEventIDHighRiskIDs:    getenvUint32Set("NEW_EVENT_ID_HIGH_RISK_IDS", "1102,4697,4719,7045"),
+
 		EnrollmentKey: mustGetenv("ENROLLMENT_KEY"),

 		BaselineEnabled:      getenvBool("BASELINE_ENABLED", true),
@@ -3997,6 +4016,55 @@ func loadConfig() Config {
 	}
 }

+func getenvCSV(key, def string) []string {
+	v := strings.TrimSpace(os.Getenv(key))
+	if v == "" {
+		v = def
+	}
+	if strings.TrimSpace(v) == "" {
+		return nil
+	}
+	parts := strings.Split(v, ",")
+	out := make([]string, 0, len(parts))
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if part != "" {
+			out = append(out, part)
+		}
+	}
+	return out
+}
+
+func getenvUint32Set(key, def string) map[uint32]struct{} {
+	values := getenvCSV(key, def)
+	out := make(map[uint32]struct{}, len(values))
+	for _, value := range values {
+		n, err := strconv.ParseUint(value, 10, 32)
+		if err != nil || n == 0 {
+			log.Fatalf("invalid event id %q in %s", value, key)
+		}
+		out[uint32(n)] = struct{}{}
+	}
+	return out
+}
+
+func containsFold(values []string, candidate string) bool {
+	candidate = strings.TrimSpace(candidate)
+	for _, value := range values {
+		if strings.EqualFold(strings.TrimSpace(value), candidate) {
+			return true
+		}
+	}
+	return false
+}
+
+func maxInt(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
 func getenvBool(key string, def bool) bool {
 	v := strings.TrimSpace(os.Getenv(key))
 	if v == "" {
@@ -6086,16 +6154,32 @@ HAVING SUM(cnt) >= ?
 }

 func (d *detector) runNewEventIDRule(ctx context.Context) error {
+	mode := strings.ToLower(strings.TrimSpace(d.cfg.NewEventIDMode))
+	if mode == "" || mode == "off" || mode == "disabled" {
+		return nil
+	}
+	if mode != "inventory" && mode != "alert" {
+		return fmt.Errorf("invalid NEW_EVENT_ID_MODE %q (expected off, inventory or alert)", d.cfg.NewEventIDMode)
+	}
+
 	windowEnd := time.Now().UTC()
-	windowStart := windowEnd.Add(-d.cfg.DetectionInterval)
+	confirmWindow := d.cfg.NewEventIDConfirmWindow
+	if confirmWindow <= 0 {
+		confirmWindow = 15 * time.Minute
+	}
+	windowStart := windowEnd.Add(-confirmWindow)

 	const q = `
-SELECT hostname, channel_name, event_id, total_count
-FROM event_catalog
-WHERE first_seen >= ? AND first_seen < ?
-ORDER BY first_seen ASC
+SELECT ec.hostname, ec.channel_name, ec.event_id, ec.total_count,
+       ec.first_seen, ec.last_seen,
+       COALESCE(a.first_seen, TIMESTAMP('1970-01-01 00:00:00')) AS agent_first_seen
+FROM event_catalog ec
+LEFT JOIN agents a ON a.hostname = ec.hostname
+WHERE ec.first_seen >= ? AND ec.first_seen < ?
+  AND ec.total_count >= ?
+ORDER BY ec.first_seen ASC
 `
-	rows, err := d.db.QueryContext(ctx, q, windowStart, windowEnd)
+	rows, err := d.db.QueryContext(ctx, q, windowStart, windowEnd, maxInt(d.cfg.NewEventIDMinCount, 1))
 	if err != nil {
 		return err
 	}
@@ -6105,43 +6189,107 @@ ORDER BY first_seen ASC
 		var host, channel string
 		var eventID uint32
 		var count int
-		if err := rows.Scan(&host, &channel, &eventID, &count); err != nil {
+		var eventFirstSeen, eventLastSeen, agentFirstSeen time.Time
+		if err := rows.Scan(&host, &channel, &eventID, &count, &eventFirstSeen, &eventLastSeen, &agentFirstSeen); err != nil {
 			return err
 		}

-		score := 1.0 + math.Log10(float64(count)+1)
-		severity := "medium"
-		if count >= 10 {
-			severity = "high"
+		assessment := assessNewEventID(d.cfg, channel, eventID, count, agentFirstSeen, eventFirstSeen)
+		if !assessment.Emit {
+			continue
+		}
+
+		d.anomalyScoreGauge.WithLabelValues(host, "new_event_id").Set(assessment.Score)
+
+		summary := fmt.Sprintf("Neue Event-ID im Inventar: Host %s, Event-ID %d, Channel %s", host, eventID, channel)
+		if assessment.Status == "open" {
+			summary = fmt.Sprintf("Ungewohnte Event-ID: Host %s, Event-ID %d, Channel %s", host, eventID, channel)
 		}
-		d.anomalyScoreGauge.WithLabelValues(host, "new_event_id").Set(score)

 		created, err := d.insertDetection(ctx, Detection{
-			RuleName:    "new_event_id",
-			Severity:    severity,
-			Hostname:    host,
-			Channel:     channel,
-			EventID:     eventID,
-			Score:       score,
-			WindowStart: windowStart,
-			WindowEnd:   windowEnd,
-			Summary:     fmt.Sprintf("Host %s sendet erstmals Event-ID %d im Channel %s", host, eventID, channel),
+			RuleName: "new_event_id",
+			Severity: assessment.Severity,
+			Status:   assessment.Status,
+			Hostname: host,
+			Channel:  channel,
+			EventID:  eventID,
+			Score:    assessment.Score,
+			// Stable per Event-ID. This prevents a new duplicate on every rule run.
+			WindowStart: eventFirstSeen.UTC(),
+			WindowEnd:   eventFirstSeen.UTC().Add(confirmWindow),
+			Summary:     summary,
 			Details: mustJSON(map[string]any{
-				"count":    count,
-				"channel":  channel,
-				"event_id": eventID,
+				"classification":       assessment.Reason,
+				"mode":                 mode,
+				"count":                count,
+				"channel":              channel,
+				"event_id":             eventID,
+				"event_first_seen":     eventFirstSeen.UTC(),
+				"event_last_seen":      eventLastSeen.UTC(),
+				"agent_first_seen":     agentFirstSeen.UTC(),
+				"agent_age_at_event_h": eventFirstSeen.Sub(agentFirstSeen).Hours(),
 			}),
 		})
 		if err != nil {
 			return err
 		}
 		if created {
-			d.detectionHitsTotal.WithLabelValues("new_event_id", severity).Inc()
+			d.detectionHitsTotal.WithLabelValues("new_event_id", assessment.Severity).Inc()
 		}
 	}
 	return rows.Err()
 }

+type newEventIDAssessment struct {
+	Emit     bool
+	Severity string
+	Status   string
+	Score    float64
+	Reason   string
+}
+
+func assessNewEventID(cfg Config, channel string, eventID uint32, count int, agentFirstSeen, eventFirstSeen time.Time) newEventIDAssessment {
+	mode := strings.ToLower(strings.TrimSpace(cfg.NewEventIDMode))
+	if mode == "" || mode == "off" || mode == "disabled" {
+		return newEventIDAssessment{Reason: "rule_disabled"}
+	}
+	if count < maxInt(cfg.NewEventIDMinCount, 1) {
+		return newEventIDAssessment{Reason: "below_confirmation_count"}
+	}
+	if cfg.NewEventIDLearningPeriod > 0 && eventFirstSeen.Sub(agentFirstSeen) < cfg.NewEventIDLearningPeriod {
+		return newEventIDAssessment{Reason: "agent_learning_period"}
+	}
+	if containsFold(cfg.NewEventIDIgnoreChannels, channel) {
+		return newEventIDAssessment{Reason: "ignored_operational_channel"}
+	}
+
+	score := 0.25 + math.Log10(float64(count)+1)/2
+	if score > 5 {
+		score = 5
+	}
+
+	if mode == "inventory" {
+		return newEventIDAssessment{true, "info", "plausible", score, "inventory_only"}
+	}
+
+	_, highRisk := cfg.NewEventIDHighRiskIDs[eventID]
+	if highRisk {
+		severity := "medium"
+		if count >= 10 {
+			severity = "high"
+		}
+		return newEventIDAssessment{true, severity, "open", score + 1, "configured_high_risk_event_id"}
+	}
+	if containsFold(cfg.NewEventIDAlertChannels, channel) {
+		severity := "low"
+		if count >= 20 {
+			severity = "medium"
+		}
+		return newEventIDAssessment{true, severity, "open", score, "new_id_in_alert_channel"}
+	}
+	return newEventIDAssessment{true, "info", "plausible", score, "new_id_in_non_alert_channel"}
+}
+
 func (d *detector) runPasswordSprayRule(ctx context.Context) error {
 	windowEnd := time.Now().UTC()
 	windowStart := windowEnd.Add(-d.cfg.PasswordSprayWindow)