From f9662780f7f7174d721f7f98c0aeaf149c921eea Mon Sep 17 00:00:00 2001 From: jbergner Date: Sat, 18 Jul 2026 19:46:02 +0200 Subject: [PATCH] Update mit Stresstest --- README.md | 70 +++++++++++++++++++ compose.yml | 33 +++++++++ dot_env | 22 ++++++ main.go | 198 +++++++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 298 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 623aaec..db92b25 100644 --- a/README.md +++ b/README.md @@ -4614,3 +4614,73 @@ Besonders wichtig für die aktuelle `runOffHoursLoginRule`: SYSTEM, LOCAL SERVICE, NETWORK SERVICE, ANONYMOUS LOGON und Maschinenkonten sollten ignoriert werden. Zusätzlich sollte die Regel auf LogonType 2, 7, 10 und 11 begrenzt werden. ``` + +--- + +## Metadata-first Ergänzung: neue Event-IDs realistisch bewerten + +Eine bislang unbekannte Event-ID ist zunächst eine Änderung des Event-Inventars und noch kein Sicherheitsvorfall. Die Regel `new_event_id` besitzt deshalb nun drei Betriebsarten: + +- `NEW_EVENT_ID_MODE=off`: Regel vollständig deaktiviert. `event_catalog` wird weiterhin gepflegt. +- `NEW_EVENT_ID_MODE=inventory` (Standard): bestätigte neue IDs werden höchstens als `info` mit Status `plausible` abgelegt. +- `NEW_EVENT_ID_MODE=alert`: neue IDs in den konfigurierten Alert-Channels werden als `low` beziehungsweise bei hoher Wiederholung als `medium` geöffnet. Erst im Modus `alert` werden explizit konfigurierte High-Risk-IDs zu `medium`/`high` hochgestuft. + +Zusätzliche Schutzmechanismen: + +```env +NEW_EVENT_ID_LEARNING_PERIOD=24h +NEW_EVENT_ID_CONFIRM_WINDOW=15m +NEW_EVENT_ID_MIN_COUNT=3 +NEW_EVENT_ID_IGNORE_CHANNELS=Microsoft-Windows-WMI-Activity/Operational +NEW_EVENT_ID_ALERT_CHANNELS=Security,System +NEW_EVENT_ID_HIGH_RISK_IDS=1102,4697,4719,7045 +``` + +Damit erzeugt Event-ID `5857` im Channel `Microsoft-Windows-WMI-Activity/Operational` standardmäßig keine Detection mehr. Neue Agenten dürfen zunächst eine Baseline lernen. Ein einmaliges Event wird ebenfalls nicht gemeldet. Die stabile Detection-Zeitspanne basiert auf `event_catalog.first_seen`, sodass die Regel bei wiederholten Läufen keine neuen Duplikate erzeugt. + +Für bestehende Installationen kann anschließend ausgeführt werden: + +```bash +mariadb -u root -p eventcollector < deploy/mariadb/migrations/003-new-event-id-classification.sql +``` + +Die Migration stuft bereits offene `new_event_id`-Meldungen aus dem WMI-Activity/Operational-Channel als `info` und `legitimate` ein. + +## Begrenzter Stress-Test-Agent + +Der Agent erzeugt ausschließlich klar markierte Metadaten (`source` und `provider` sind `SIEM-Stress-Agent`). Er besitzt harte Grenzen für Rate, Laufzeit, Worker und Gesamtzahl und startet nur mit expliziter Bestätigung. + +Direkt mit Go: + +```bash +go run ./cmd/siem-stress-agent \ + --url http://127.0.0.1:8090/ingest \ + --host SIEM-STRESS-01 \ + --api-key 'EIN-EIGENER-TEST-API-KEY' \ + --enrollment-key 'DER-ENROLLMENT-KEY' \ + --scenario mixed \ + --rate 500 \ + --batch 100 \ + --workers 4 \ + --duration 2m \ + --max-events 60000 \ + --confirm-load-test +``` + +Über das opt-in Compose-Profil: + +```bash +cp .env.example .env +# Kennwörter, ENROLLMENT_KEY und SIEM_STRESS_API_KEY setzen +docker compose --profile stress run --rm stress-agent +``` + +Szenarien: + +- `normal`: erfolgreiche Logons, um den normalen Ingest-Durchsatz zu messen. +- `failed-logon`: Event 4625 mit wechselnden Benutzern und Test-Netz-IP-Adressen. +- `lockout`: Event 4740 mit Testbenutzern und Test-Workstations. +- `mixed`: Mischung aus normalen Logons, Fehlern, Lockouts, Reboots und WMI 5857. +- `catalog`: bis zu 5.000 künstliche Event-IDs im eigenen Channel `SIEM-Stress-Agent/Operational`; geeignet zum Test des Event-Katalogs, nicht für einen normalen Lasttest. + +Der Agent gibt akzeptierte/fehlgeschlagene Events, tatsächlichen Durchsatz, HTTP-Status sowie p50/p95/p99-Latenzen aus. Er sollte nur gegen ein System ausgeführt werden, für das eine ausdrückliche Lasttest-Freigabe besteht. diff --git a/compose.yml b/compose.yml index 2022a76..7c62080 100644 --- a/compose.yml +++ b/compose.yml @@ -62,6 +62,13 @@ services: NEW_SOURCE_IP_LOOKBACK: ${NEW_SOURCE_IP_LOOKBACK} NEW_SOURCE_IP_WINDOW: ${NEW_SOURCE_IP_WINDOW} DETECTIONS_LIMIT: ${DETECTIONS_LIMIT} + NEW_EVENT_ID_MODE: ${NEW_EVENT_ID_MODE} + NEW_EVENT_ID_LEARNING_PERIOD: ${NEW_EVENT_ID_LEARNING_PERIOD} + NEW_EVENT_ID_CONFIRM_WINDOW: ${NEW_EVENT_ID_CONFIRM_WINDOW} + NEW_EVENT_ID_MIN_COUNT: ${NEW_EVENT_ID_MIN_COUNT} + NEW_EVENT_ID_IGNORE_CHANNELS: ${NEW_EVENT_ID_IGNORE_CHANNELS} + NEW_EVENT_ID_ALERT_CHANNELS: ${NEW_EVENT_ID_ALERT_CHANNELS} + NEW_EVENT_ID_HIGH_RISK_IDS: ${NEW_EVENT_ID_HIGH_RISK_IDS} STORE_RAW_XML: ${STORE_RAW_XML} METADATA_BUCKET: ${METADATA_BUCKET} EVENT_RETENTION: ${EVENT_RETENTION} @@ -89,6 +96,32 @@ services: start_period: 20s networks: - dockge_default + stress-agent: + profiles: ["stress"] + build: + context: . + dockerfile: Dockerfile.stress + container_name: siem-stress-agent + restart: "no" + environment: + SIEM_STRESS_URL: ${SIEM_STRESS_URL} + SIEM_STRESS_HOST: ${SIEM_STRESS_HOST} + SIEM_STRESS_API_KEY: ${SIEM_STRESS_API_KEY} + SIEM_STRESS_ENROLLMENT_KEY: ${ENROLLMENT_KEY} + SIEM_STRESS_SCENARIO: ${SIEM_STRESS_SCENARIO} + SIEM_STRESS_RATE: ${SIEM_STRESS_RATE} + SIEM_STRESS_BATCH: ${SIEM_STRESS_BATCH} + SIEM_STRESS_WORKERS: ${SIEM_STRESS_WORKERS} + SIEM_STRESS_DURATION: ${SIEM_STRESS_DURATION} + SIEM_STRESS_MAX_EVENTS: ${SIEM_STRESS_MAX_EVENTS} + SIEM_STRESS_TIMEOUT: ${SIEM_STRESS_TIMEOUT} + SIEM_STRESS_CONFIRM: ${SIEM_STRESS_CONFIRM} + depends_on: + siem-backend: + condition: service_healthy + networks: + - dockge_default + prometheus: image: prom/prometheus:latest container_name: siem-prometheus diff --git a/dot_env b/dot_env index 4e96437..d782a17 100644 --- a/dot_env +++ b/dot_env @@ -67,3 +67,25 @@ PARTITION_AHEAD=24h PARTITION_BEHIND=6h # Compatibility fallback; table-specific retention values above take precedence. PARTITION_RETENTION=720h + +# New Event-ID classification +NEW_EVENT_ID_MODE=inventory +NEW_EVENT_ID_LEARNING_PERIOD=24h +NEW_EVENT_ID_CONFIRM_WINDOW=15m +NEW_EVENT_ID_MIN_COUNT=3 +NEW_EVENT_ID_IGNORE_CHANNELS=Microsoft-Windows-WMI-Activity/Operational +NEW_EVENT_ID_ALERT_CHANNELS=Security,System +NEW_EVENT_ID_HIGH_RISK_IDS=1102,4697,4719,7045 + +# Optional stress-agent profile +SIEM_STRESS_URL=http://siem-backend:8080/ingest +SIEM_STRESS_HOST=SIEM-STRESS-01 +SIEM_STRESS_API_KEY=CHANGE-ME-STRESS-API-KEY +SIEM_STRESS_SCENARIO=mixed +SIEM_STRESS_RATE=200 +SIEM_STRESS_BATCH=100 +SIEM_STRESS_WORKERS=4 +SIEM_STRESS_DURATION=30s +SIEM_STRESS_MAX_EVENTS=100000 +SIEM_STRESS_TIMEOUT=20s +SIEM_STRESS_CONFIRM=true diff --git a/main.go b/main.go index d5ba649..83eca61 100644 --- a/main.go +++ b/main.go @@ -1149,6 +1149,17 @@ type Config struct { NewSourceIPWindow time.Duration DetectionsLimit int + // A previously unseen Event ID is primarily inventory information, not an + // incident. The rule can be disabled, run in inventory mode, or explicitly + // promoted to alert mode for selected channels/Event IDs. + NewEventIDMode string + NewEventIDLearningPeriod time.Duration + NewEventIDConfirmWindow time.Duration + NewEventIDMinCount int + NewEventIDIgnoreChannels []string + NewEventIDAlertChannels []string + NewEventIDHighRiskIDs map[uint32]struct{} + EnrollmentKey string BaselineEnabled bool @@ -3965,6 +3976,14 @@ func loadConfig() Config { NewSourceIPWindow: getenvDuration("NEW_SOURCE_IP_WINDOW", 10*time.Minute), DetectionsLimit: getenvInt("DETECTIONS_LIMIT", 100), + NewEventIDMode: strings.ToLower(getenv("NEW_EVENT_ID_MODE", "inventory")), + NewEventIDLearningPeriod: getenvDuration("NEW_EVENT_ID_LEARNING_PERIOD", 24*time.Hour), + NewEventIDConfirmWindow: getenvDuration("NEW_EVENT_ID_CONFIRM_WINDOW", 15*time.Minute), + NewEventIDMinCount: getenvInt("NEW_EVENT_ID_MIN_COUNT", 3), + NewEventIDIgnoreChannels: getenvCSV("NEW_EVENT_ID_IGNORE_CHANNELS", "Microsoft-Windows-WMI-Activity/Operational"), + NewEventIDAlertChannels: getenvCSV("NEW_EVENT_ID_ALERT_CHANNELS", "Security,System"), + NewEventIDHighRiskIDs: getenvUint32Set("NEW_EVENT_ID_HIGH_RISK_IDS", "1102,4697,4719,7045"), + EnrollmentKey: mustGetenv("ENROLLMENT_KEY"), BaselineEnabled: getenvBool("BASELINE_ENABLED", true), @@ -3997,6 +4016,55 @@ func loadConfig() Config { } } +func getenvCSV(key, def string) []string { + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + v = def + } + if strings.TrimSpace(v) == "" { + return nil + } + parts := strings.Split(v, ",") + out := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + +func getenvUint32Set(key, def string) map[uint32]struct{} { + values := getenvCSV(key, def) + out := make(map[uint32]struct{}, len(values)) + for _, value := range values { + n, err := strconv.ParseUint(value, 10, 32) + if err != nil || n == 0 { + log.Fatalf("invalid event id %q in %s", value, key) + } + out[uint32(n)] = struct{}{} + } + return out +} + +func containsFold(values []string, candidate string) bool { + candidate = strings.TrimSpace(candidate) + for _, value := range values { + if strings.EqualFold(strings.TrimSpace(value), candidate) { + return true + } + } + return false +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} + func getenvBool(key string, def bool) bool { v := strings.TrimSpace(os.Getenv(key)) if v == "" { @@ -6086,16 +6154,32 @@ HAVING SUM(cnt) >= ? } func (d *detector) runNewEventIDRule(ctx context.Context) error { + mode := strings.ToLower(strings.TrimSpace(d.cfg.NewEventIDMode)) + if mode == "" || mode == "off" || mode == "disabled" { + return nil + } + if mode != "inventory" && mode != "alert" { + return fmt.Errorf("invalid NEW_EVENT_ID_MODE %q (expected off, inventory or alert)", d.cfg.NewEventIDMode) + } + windowEnd := time.Now().UTC() - windowStart := windowEnd.Add(-d.cfg.DetectionInterval) + confirmWindow := d.cfg.NewEventIDConfirmWindow + if confirmWindow <= 0 { + confirmWindow = 15 * time.Minute + } + windowStart := windowEnd.Add(-confirmWindow) const q = ` -SELECT hostname, channel_name, event_id, total_count -FROM event_catalog -WHERE first_seen >= ? AND first_seen < ? -ORDER BY first_seen ASC +SELECT ec.hostname, ec.channel_name, ec.event_id, ec.total_count, + ec.first_seen, ec.last_seen, + COALESCE(a.first_seen, TIMESTAMP('1970-01-01 00:00:00')) AS agent_first_seen +FROM event_catalog ec +LEFT JOIN agents a ON a.hostname = ec.hostname +WHERE ec.first_seen >= ? AND ec.first_seen < ? + AND ec.total_count >= ? +ORDER BY ec.first_seen ASC ` - rows, err := d.db.QueryContext(ctx, q, windowStart, windowEnd) + rows, err := d.db.QueryContext(ctx, q, windowStart, windowEnd, maxInt(d.cfg.NewEventIDMinCount, 1)) if err != nil { return err } @@ -6105,43 +6189,107 @@ ORDER BY first_seen ASC var host, channel string var eventID uint32 var count int - if err := rows.Scan(&host, &channel, &eventID, &count); err != nil { + var eventFirstSeen, eventLastSeen, agentFirstSeen time.Time + if err := rows.Scan(&host, &channel, &eventID, &count, &eventFirstSeen, &eventLastSeen, &agentFirstSeen); err != nil { return err } - score := 1.0 + math.Log10(float64(count)+1) - severity := "medium" - if count >= 10 { - severity = "high" + assessment := assessNewEventID(d.cfg, channel, eventID, count, agentFirstSeen, eventFirstSeen) + if !assessment.Emit { + continue + } + + d.anomalyScoreGauge.WithLabelValues(host, "new_event_id").Set(assessment.Score) + + summary := fmt.Sprintf("Neue Event-ID im Inventar: Host %s, Event-ID %d, Channel %s", host, eventID, channel) + if assessment.Status == "open" { + summary = fmt.Sprintf("Ungewohnte Event-ID: Host %s, Event-ID %d, Channel %s", host, eventID, channel) } - d.anomalyScoreGauge.WithLabelValues(host, "new_event_id").Set(score) created, err := d.insertDetection(ctx, Detection{ - RuleName: "new_event_id", - Severity: severity, - Hostname: host, - Channel: channel, - EventID: eventID, - Score: score, - WindowStart: windowStart, - WindowEnd: windowEnd, - Summary: fmt.Sprintf("Host %s sendet erstmals Event-ID %d im Channel %s", host, eventID, channel), + RuleName: "new_event_id", + Severity: assessment.Severity, + Status: assessment.Status, + Hostname: host, + Channel: channel, + EventID: eventID, + Score: assessment.Score, + // Stable per Event-ID. This prevents a new duplicate on every rule run. + WindowStart: eventFirstSeen.UTC(), + WindowEnd: eventFirstSeen.UTC().Add(confirmWindow), + Summary: summary, Details: mustJSON(map[string]any{ - "count": count, - "channel": channel, - "event_id": eventID, + "classification": assessment.Reason, + "mode": mode, + "count": count, + "channel": channel, + "event_id": eventID, + "event_first_seen": eventFirstSeen.UTC(), + "event_last_seen": eventLastSeen.UTC(), + "agent_first_seen": agentFirstSeen.UTC(), + "agent_age_at_event_h": eventFirstSeen.Sub(agentFirstSeen).Hours(), }), }) if err != nil { return err } if created { - d.detectionHitsTotal.WithLabelValues("new_event_id", severity).Inc() + d.detectionHitsTotal.WithLabelValues("new_event_id", assessment.Severity).Inc() } } return rows.Err() } +type newEventIDAssessment struct { + Emit bool + Severity string + Status string + Score float64 + Reason string +} + +func assessNewEventID(cfg Config, channel string, eventID uint32, count int, agentFirstSeen, eventFirstSeen time.Time) newEventIDAssessment { + mode := strings.ToLower(strings.TrimSpace(cfg.NewEventIDMode)) + if mode == "" || mode == "off" || mode == "disabled" { + return newEventIDAssessment{Reason: "rule_disabled"} + } + if count < maxInt(cfg.NewEventIDMinCount, 1) { + return newEventIDAssessment{Reason: "below_confirmation_count"} + } + if cfg.NewEventIDLearningPeriod > 0 && eventFirstSeen.Sub(agentFirstSeen) < cfg.NewEventIDLearningPeriod { + return newEventIDAssessment{Reason: "agent_learning_period"} + } + if containsFold(cfg.NewEventIDIgnoreChannels, channel) { + return newEventIDAssessment{Reason: "ignored_operational_channel"} + } + + score := 0.25 + math.Log10(float64(count)+1)/2 + if score > 5 { + score = 5 + } + + if mode == "inventory" { + return newEventIDAssessment{true, "info", "plausible", score, "inventory_only"} + } + + _, highRisk := cfg.NewEventIDHighRiskIDs[eventID] + if highRisk { + severity := "medium" + if count >= 10 { + severity = "high" + } + return newEventIDAssessment{true, severity, "open", score + 1, "configured_high_risk_event_id"} + } + if containsFold(cfg.NewEventIDAlertChannels, channel) { + severity := "low" + if count >= 20 { + severity = "medium" + } + return newEventIDAssessment{true, severity, "open", score, "new_id_in_alert_channel"} + } + return newEventIDAssessment{true, "info", "plausible", score, "new_id_in_non_alert_channel"} +} + func (d *detector) runPasswordSprayRule(ctx context.Context) error { windowEnd := time.Now().UTC() windowStart := windowEnd.Add(-d.cfg.PasswordSprayWindow)