diff --git a/deploy/mariadb/init/001-schema.sql b/deploy/mariadb/init/001-schema.sql index ef06ea0..3d0b42e 100644 --- a/deploy/mariadb/init/001-schema.sql +++ b/deploy/mariadb/init/001-schema.sql @@ -1350,4 +1350,34 @@ ON baseline_event_stats ( hour_of_day, day_of_week, sample_count -); \ No newline at end of file +); + +ALTER TABLE detections +ADD COLUMN status VARCHAR(32) NOT NULL DEFAULT 'open', +ADD COLUMN analyst_note TEXT NULL, +ADD COLUMN reviewed_by VARCHAR(128) NULL, +ADD COLUMN reviewed_at TIMESTAMP(6) NULL, +ADD COLUMN is_false_positive TINYINT(1) NOT NULL DEFAULT 0, +ADD COLUMN is_legitimate TINYINT(1) NOT NULL DEFAULT 0; + +CREATE INDEX idx_detections_status_created +ON detections (status, created_at); + +CREATE INDEX idx_detections_rule_status +ON detections (rule_name, status); + +CREATE TABLE detection_suppressions ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + rule_name VARCHAR(128) NOT NULL, + hostname VARCHAR(255) DEFAULT '', + channel_name VARCHAR(255) DEFAULT '', + event_id INT DEFAULT 0, + reason TEXT NULL, + created_by VARCHAR(128) DEFAULT '', + created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + expires_at TIMESTAMP(6) NULL, + enabled TINYINT(1) NOT NULL DEFAULT 1 +); + +CREATE INDEX idx_suppressions_lookup +ON detection_suppressions (enabled, rule_name, hostname, channel_name, event_id); \ No newline at end of file diff --git a/deploy/prometheus/rules/siem-alerts.yml b/deploy/prometheus/rules/siem-alerts.yml index b6f2cee..edaeff1 100644 --- a/deploy/prometheus/rules/siem-alerts.yml +++ b/deploy/prometheus/rules/siem-alerts.yml @@ -28,8 +28,15 @@ groups: summary: "Zu wenige aktive Agents" description: "Es wurden weniger aktive Agents erkannt als erwartet." - - name: siem-backend-detections - rules: + - alert: SiemCriticalDetections + expr: increase(eventcollector_detection_hits_total{severity="critical"}[5m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Neue Critical Detection" + description: "Es wurde mindestens eine Critical-Detection erzeugt." + - alert: SiemHighDetections expr: increase(eventcollector_detection_hits_total{severity="high"}[5m]) > 0 for: 1m @@ -37,7 +44,7 @@ groups: severity: high annotations: summary: "Neue High-Severity Detection" - description: "Es wurde mindestens eine neue High-Severity-Detection in den letzten 5 Minuten erzeugt." + description: "Es wurde mindestens eine High-Severity-Detection erzeugt." - alert: SiemManyMediumDetections expr: sum(increase(eventcollector_detection_hits_total{severity="medium"}[15m])) > 10 diff --git a/main.go b/main.go index b04efb7..ed85d73 100644 --- a/main.go +++ b/main.go @@ -37,45 +37,362 @@ const uiTemplates = ` {{.Title}} @@ -110,9 +427,15 @@ const uiTemplates = `
Events 24h
{{.Stats.Events24h}}
Detections 24h
{{.Stats.Detections24h}}
High Detections 24h
{{.Stats.HighDetections24h}}
+
Open Detections
{{.Stats.OpenDetections}}
+
Investigating
{{.Stats.InvestigatingDetections}}
+
Critical 24h
{{.Stats.CriticalDetections24h}}
+
False Positive 24h
{{.Stats.FalsePositive24h}}
+
Legitim 24h
{{.Stats.Legitimate24h}}

Neueste Detections

+
{{range .RecentDetections}} @@ -125,8 +448,10 @@ const uiTemplates = ` {{end}}
ZeitRuleSeverityHostZusammenfassung
+

Neueste Events

+
{{range .RecentEvents}} @@ -141,6 +466,7 @@ const uiTemplates = ` {{end}}
ZeitHostChannelEventIDUserIPNachricht
+
{{template "footer" .}} {{end}} @@ -152,24 +478,97 @@ const uiTemplates = `
+
+ + +
+
+ Open + Critical + High + Investigating + False Positives + Legitim + Resolved +
+
- + + + + + + + + + + + {{range .Detections}} - - - - - - - - + + + + + + + + + + {{end}}
ZeitRuleSeverityHostScoreSummaryEvents
ZeitRuleSeverityStatusHostScoreSummaryBewertungEvents
{{fmtTime .CreatedAt}}{{.RuleName}}{{.Severity}}{{.Hostname}}{{printf "%.2f" .Score}}{{.Summary}}anzeigen
{{fmtTime .CreatedAt}}{{.RuleName}}{{.Severity}}{{.Status}}{{.Hostname}}{{printf "%.2f" .Score}} + {{.Summary}} + {{if .AnalystNote}} +
Notiz: {{.AnalystNote}}
+ {{end}} + 		+
+ + + + + + + + + + + +
+ 		+ anzeigen +
+
{{template "footer" .}} {{end}} @@ -210,6 +609,7 @@ const uiTemplates = `

Bestehende Regeln

+
@@ -244,6 +644,7 @@ const uiTemplates = ` {{end}}
NameSeverityChannelEventsFilterThresholdStatusAktion
+
{{template "footer" .}} {{end}} @@ -263,6 +664,7 @@ const uiTemplates = ` +
@@ -291,6 +693,7 @@ const uiTemplates = ` {{end}}
Zeit
+
{{template "footer" .}} {{end}} @@ -312,6 +715,7 @@ const uiTemplates = ` +
@@ -330,6 +734,7 @@ const uiTemplates = ` {{end}}
ZeitHostChannelEventIDTarget UserSubject UserIPWorkstationDetail
+
{{template "footer" .}} {{end}} @@ -337,7 +742,7 @@ const uiTemplates = ` {{template "header" .}}

{{.Title}}

Stand: {{fmtTime .Now}}

- +
@@ -385,6 +790,7 @@ const uiTemplates = ` {{end}}
Hostname
+
{{template "footer" .}} {{end}} @@ -496,6 +902,13 @@ type Detection struct { Summary string `json:"summary"` Details json.RawMessage `json:"details_json"` CreatedAt time.Time `json:"created_at"` + + Status string + AnalystNote string + ReviewedBy string + ReviewedAt sql.NullTime + IsFalsePositive bool + IsLegitimate bool } type ingestResponse struct { @@ -564,6 +977,12 @@ type DashboardStats struct { Events24h int64 Detections24h int64 HighDetections24h int64 + + OpenDetections int64 + InvestigatingDetections int64 + CriticalDetections24h int64 + FalsePositive24h int64 + Legitimate24h int64 } type DashboardPageData struct { @@ -857,6 +1276,9 @@ func main() { } return s[:n] + "..." }, + "eq": func(a, b string) bool { + return a == b + }, }).Parse(uiTemplates)) s.templates = tmpl @@ -871,6 +1293,7 @@ func main() { mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) mux.HandleFunc("/ui", s.handleUIIndex) mux.HandleFunc("/ui/detections", s.handleUIDetections) + mux.HandleFunc("/ui/detection/update", s.handleUIDetectionUpdate) mux.HandleFunc("/ui/events", s.handleUIEvents) mux.HandleFunc("/ui/event", s.handleUIEventDetail) mux.HandleFunc("/ui/agents", s.handleUIAgents) @@ -912,6 +1335,161 @@ func main() { } } +func (s *server) handleUIDetectionUpdate(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + + if err := r.ParseForm(); err != nil { + writeError(w, http.StatusBadRequest, "invalid form") + return + } + + id, err := strconv.ParseUint(strings.TrimSpace(r.FormValue("id")), 10, 64) + if err != nil || id == 0 { + writeError(w, http.StatusBadRequest, "invalid id") + return + } + + status := strings.TrimSpace(r.FormValue("status")) + note := strings.TrimSpace(r.FormValue("note")) + reviewedBy := strings.TrimSpace(r.FormValue("reviewed_by")) + + if reviewedBy == "" { + reviewedBy = "ui" + } + + switch status { + case "open", "acknowledged", "investigating", "legitimate", "false_positive", "resolved", "suppressed": + default: + writeError(w, http.StatusBadRequest, "invalid status") + return + } + + isFalsePositive := status == "false_positive" + isLegitimate := status == "legitimate" + + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + _, err = s.db.ExecContext(ctx, ` +UPDATE detections +SET status = ?, + analyst_note = ?, + reviewed_by = ?, + reviewed_at = UTC_TIMESTAMP(6), + is_false_positive = ?, + is_legitimate = ? +WHERE id = ? +`, + status, + note, + reviewedBy, + isFalsePositive, + isLegitimate, + id, + ) + if err != nil { + s.logger.Printf("update detection: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + + createSuppression := strings.TrimSpace(r.FormValue("create_suppression")) == "1" + + if createSuppression && (status == "false_positive" || status == "legitimate" || status == "suppressed") { + det, err := s.getDetectionByID(ctx, id) + if err != nil { + s.logger.Printf("get detection for suppression: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + + hours := atoiDefault(r.FormValue("suppress_hours"), 24) + + reason := note + if reason == "" { + reason = fmt.Sprintf("Suppression via UI wegen Status %s", status) + } + + if err := s.createSuppressionFromDetection(ctx, det, reason, reviewedBy, hours); err != nil { + s.logger.Printf("create suppression: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + } + + redirect := strings.TrimSpace(r.FormValue("redirect")) + if redirect == "" { + redirect = "/ui/detections" + } + + http.Redirect(w, r, redirect, http.StatusSeeOther) +} + +func (s *server) getDetectionByID(ctx context.Context, id uint64) (Detection, error) { + const q = ` +SELECT id, rule_name, severity, hostname, channel_name, event_id, score, + window_start, window_end, summary, details_json, created_at, + status, + COALESCE(analyst_note, ''), + COALESCE(reviewed_by, ''), + reviewed_at, + is_false_positive, + is_legitimate +FROM detections +WHERE id = ? +LIMIT 1 +` + + var d Detection + err := s.db.QueryRowContext(ctx, q, id).Scan( + &d.ID, + &d.RuleName, + &d.Severity, + &d.Hostname, + &d.Channel, + &d.EventID, + &d.Score, + &d.WindowStart, + &d.WindowEnd, + &d.Summary, + &d.Details, + &d.CreatedAt, + &d.Status, + &d.AnalystNote, + &d.ReviewedBy, + &d.ReviewedAt, + &d.IsFalsePositive, + &d.IsLegitimate, + ) + return d, err +} + +func (s *server) createSuppressionFromDetection(ctx context.Context, det Detection, reason, createdBy string, hours int) error { + var expiresAt any = nil + if hours > 0 { + expiresAt = time.Now().UTC().Add(time.Duration(hours) * time.Hour) + } + + _, err := s.db.ExecContext(ctx, ` +INSERT INTO detection_suppressions +(rule_name, hostname, channel_name, event_id, reason, created_by, expires_at, enabled) +VALUES (?, ?, ?, ?, ?, ?, ?, 1) +`, + det.RuleName, + det.Hostname, + det.Channel, + det.EventID, + reason, + createdBy, + expiresAt, + ) + + return err +} + func (s *server) listBaselineAnomalies(ctx context.Context, host, channel, severity string, eventID uint32, limit int) ([]BaselineAnomalyRow, error) { if limit <= 0 || limit > 1000 { limit = 100 @@ -1377,7 +1955,7 @@ func (s *server) handleUIIndex(w http.ResponseWriter, r *http.Request) { return } - dets, err := s.listDetections(ctx, "", "", "", 20) + dets, err := s.listDetections(ctx, "", "", "", "", 20) if err != nil { s.logger.Printf("dashboard detections: %v", err) writeError(w, http.StatusInternalServerError, "internal error") @@ -1412,6 +1990,7 @@ func (s *server) handleUIDetections(w http.ResponseWriter, r *http.Request) { "host": strings.TrimSpace(r.URL.Query().Get("host")), "rule": strings.TrimSpace(r.URL.Query().Get("rule")), "severity": strings.TrimSpace(r.URL.Query().Get("severity")), + "status": strings.TrimSpace(r.URL.Query().Get("status")), "limit": strings.TrimSpace(r.URL.Query().Get("limit")), } @@ -1425,7 +2004,7 @@ func (s *server) handleUIDetections(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) defer cancel() - items, err := s.listDetections(ctx, filters["host"], filters["rule"], filters["severity"], limit) + items, err := s.listDetections(ctx, filters["host"], filters["rule"], filters["severity"], filters["status"], limit) if err != nil { s.logger.Printf("ui detections: %v", err) writeError(w, http.StatusInternalServerError, "internal error") @@ -1590,6 +2169,39 @@ func (s *server) getDashboardStats(ctx context.Context) (DashboardStats, error) return stats, err } + if err := s.db.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM detections WHERE status = 'open' +`).Scan(&stats.OpenDetections); err != nil { + return stats, err + } + + if err := s.db.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM detections WHERE status = 'investigating' +`).Scan(&stats.InvestigatingDetections); err != nil { + return stats, err + } + + if err := s.db.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM detections +WHERE created_at >= ? AND severity = 'critical' +`, time.Now().UTC().Add(-24*time.Hour)).Scan(&stats.CriticalDetections24h); err != nil { + return stats, err + } + + if err := s.db.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM detections +WHERE created_at >= ? AND status = 'false_positive' +`, time.Now().UTC().Add(-24*time.Hour)).Scan(&stats.FalsePositive24h); err != nil { + return stats, err + } + + if err := s.db.QueryRowContext(ctx, ` +SELECT COUNT(*) FROM detections +WHERE created_at >= ? AND status = 'legitimate' +`, time.Now().UTC().Add(-24*time.Hour)).Scan(&stats.Legitimate24h); err != nil { + return stats, err + } + return stats, nil } @@ -1923,11 +2535,11 @@ func (s *server) handleDetections(w http.ResponseWriter, r *http.Request) { host := strings.TrimSpace(r.URL.Query().Get("host")) rule := strings.TrimSpace(r.URL.Query().Get("rule")) severity := strings.TrimSpace(r.URL.Query().Get("severity")) - + status := strings.TrimSpace(r.URL.Query().Get("status")) ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) defer cancel() - items, err := s.listDetections(ctx, host, rule, severity, limit) + items, err := s.listDetections(ctx, host, rule, severity, status, limit) if err != nil { s.logger.Printf("list detections: %v", err) writeError(w, http.StatusInternalServerError, "internal error") @@ -2097,10 +2709,16 @@ INSERT INTO event_logs ( return nil } -func (s *server) listDetections(ctx context.Context, host, rule, severity string, limit int) ([]Detection, error) { +func (s *server) listDetections(ctx context.Context, host, rule, severity, status string, limit int) ([]Detection, error) { base := ` SELECT id, rule_name, severity, hostname, channel_name, event_id, score, - window_start, window_end, summary, details_json, created_at + window_start, window_end, summary, details_json, created_at, + status, + COALESCE(analyst_note, ''), + COALESCE(reviewed_by, ''), + reviewed_at, + is_false_positive, + is_legitimate FROM detections WHERE 1=1 ` @@ -2118,6 +2736,10 @@ WHERE 1=1 base += " AND severity = ?" args = append(args, severity) } + if status != "" { + base += " AND status = ?" + args = append(args, status) + } base += " ORDER BY created_at DESC LIMIT ?" args = append(args, limit) @@ -2132,9 +2754,24 @@ WHERE 1=1 for rows.Next() { var d Detection if err := rows.Scan( - &d.ID, &d.RuleName, &d.Severity, &d.Hostname, &d.Channel, - &d.EventID, &d.Score, &d.WindowStart, &d.WindowEnd, - &d.Summary, &d.Details, &d.CreatedAt, + &d.ID, + &d.RuleName, + &d.Severity, + &d.Hostname, + &d.Channel, + &d.EventID, + &d.Score, + &d.WindowStart, + &d.WindowEnd, + &d.Summary, + &d.Details, + &d.CreatedAt, + &d.Status, + &d.AnalystNote, + &d.ReviewedBy, + &d.ReviewedAt, + &d.IsFalsePositive, + &d.IsLegitimate, ); err != nil { return nil, err } @@ -2965,10 +3602,11 @@ func (s *server) runDetectionsOnce() { func (d *detector) updateAgentMetrics(ctx context.Context) error { const q = ` -SELECT hostname, UNIX_TIMESTAMP(last_seen) +SELECT hostname, last_seen FROM agents WHERE is_enabled = 1 ` + rows, err := d.db.QueryContext(ctx, q) if err != nil { return err @@ -2980,20 +3618,25 @@ WHERE is_enabled = 1 for rows.Next() { var host string - var lastSeen sql.NullInt64 + var lastSeen time.Time + if err := rows.Scan(&host, &lastSeen); err != nil { return err } - if lastSeen.Valid { - d.lastSeenGauge.WithLabelValues(host).Set(float64(lastSeen.Int64)) - if now.Sub(time.Unix(lastSeen.Int64, 0).UTC()) <= d.cfg.OfflineAfter { - active++ - } + + lastSeenUTC := lastSeen.UTC() + + d.lastSeenGauge.WithLabelValues(host).Set(float64(lastSeenUTC.Unix())) + + if now.Sub(lastSeenUTC) <= d.cfg.OfflineAfter { + active++ } } + if err := rows.Err(); err != nil { return err } + d.activeAgentsGauge.Set(float64(active)) return nil } @@ -3085,7 +3728,7 @@ HAVING COUNT(*) >= ? } score := float64(count) / float64(d.cfg.FailedLogonThreshold) - severity := severityFromScore(score, 2.0, 5.0) + severity := severityFromScore(score, 1.0, 2.0, 4.0, 8.0) d.anomalyScoreGauge.WithLabelValues(host, "failed_logon_spike").Set(score) created, err := d.insertDetection(ctx, Detection{ @@ -3142,7 +3785,7 @@ HAVING COUNT(*) >= ? } score := float64(count) / float64(d.cfg.RebootThreshold) - severity := severityFromScore(score, 2.0, 4.0) + severity := severityFromScore(score, 1.0, 2.0, 4.0, 8.0) d.anomalyScoreGauge.WithLabelValues(host, "reboot_spike").Set(score) created, err := d.insertDetection(ctx, Detection{ @@ -3265,7 +3908,7 @@ HAVING COUNT(*) >= ? AND COUNT(DISTINCT target_user) >= ? } score := math.Max(float64(attempts)/float64(d.cfg.PasswordSprayMinAttempts), float64(users)/float64(d.cfg.PasswordSprayMinUsers)) - severity := severityFromScore(score, 1.5, 3.0) + severity := severityFromScore(score, 1.0, 2.0, 4.0, 8.0) d.anomalyScoreGauge.WithLabelValues(host, "password_spray").Set(score) created, err := d.insertDetection(ctx, Detection{ @@ -3443,7 +4086,42 @@ GROUP BY e.hostname, e.target_user, e.src_ip return rows.Err() } +func (d *detector) isDetectionSuppressed(ctx context.Context, det Detection) (bool, error) { + var count int + + err := d.db.QueryRowContext(ctx, ` +SELECT COUNT(*) +FROM detection_suppressions +WHERE enabled = 1 + AND rule_name = ? + AND (hostname = '' OR hostname = ?) + AND (channel_name = '' OR channel_name = ?) + AND (event_id = 0 OR event_id = ?) + AND (expires_at IS NULL OR expires_at > UTC_TIMESTAMP(6)) +`, + det.RuleName, + det.Hostname, + det.Channel, + det.EventID, + ).Scan(&count) + + if err != nil { + return false, err + } + + return count > 0, nil +} + func (d *detector) insertDetection(ctx context.Context, det Detection) (bool, error) { + + suppressed, err := d.isDetectionSuppressed(ctx, det) + if err != nil { + return false, err + } + if suppressed { + return false, nil + } + const q = ` INSERT IGNORE INTO detections (rule_name, severity, hostname, channel_name, event_id, score, window_start, window_end, summary, details_json) @@ -3683,14 +4361,18 @@ func validatePayload(p *LogPayload) error { return nil } -func severityFromScore(score, medium, high float64) string { +func severityFromScore(score, low, medium, high, critical float64) string { switch { + case score >= critical: + return "critical" case score >= high: return "high" case score >= medium: return "medium" - default: + case score >= low: return "low" + default: + return "info" } }