package rules import ( "encoding/json" "fmt" "os" "path/filepath" "sort" "strconv" "strings" "time" "example.com/siem-greenfield/internal/clickhouse" ) type RuleSet struct { ID string `json:"id"` Name string `json:"name"` Description string `json:"description"` Version int `json:"version"` Enabled bool `json:"enabled"` Rules []Rule `json:"rules"` } type Rule struct { ID string `json:"id"` Title string `json:"title"` Description string `json:"description"` Severity string `json:"severity"` Score float64 `json:"score"` Enabled bool `json:"enabled"` Kind string `json:"kind"` Channels []string `json:"channels,omitempty"` EventCodes []uint32 `json:"event_codes,omitempty"` Conditions []Condition `json:"conditions,omitempty"` GroupBy []string `json:"group_by,omitempty"` Threshold int64 `json:"threshold,omitempty"` DistinctField string `json:"distinct_field,omitempty"` DistinctThreshold int64 `json:"distinct_threshold,omitempty"` WindowSeconds int `json:"window_seconds"` SuppressSeconds int `json:"suppress_seconds,omitempty"` Summary string `json:"summary"` Tags []string `json:"tags,omitempty"` MITRE []string `json:"mitre,omitempty"` } type Condition struct { Field string `json:"field"` Operator string `json:"operator"` Value string `json:"value,omitempty"` } type StoredRule struct { RuleSetID string `json:"rule_set_id"` RuleSetName string `json:"rule_set_name"` RuleSetOn bool `json:"rule_set_enabled"` Source string `json:"source"` Rule } func LoadDir(dir string) ([]RuleSet, error) { entries, err := os.ReadDir(dir) if err != nil { return nil, err } var out []RuleSet for _, ent := range entries { if ent.IsDir() || !strings.HasSuffix(strings.ToLower(ent.Name()), ".json") { continue } b, err := os.ReadFile(filepath.Join(dir, ent.Name())) if err != nil { return nil, err } var rs RuleSet if err := json.Unmarshal(b, &rs); err != nil { return nil, fmt.Errorf("%s: %w", ent.Name(), err) } if err := ValidateSet(rs); err != nil { return nil, fmt.Errorf("%s: %w", ent.Name(), err) } out = append(out, rs) } sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) return out, nil } func ValidateSet(rs RuleSet) error { if !validID(rs.ID) || strings.TrimSpace(rs.Name) == "" { return fmt.Errorf("invalid rule-set id/name") } if rs.Version <= 0 { return fmt.Errorf("rule-set %s: version must be positive", rs.ID) } seen := map[string]bool{} for i := range rs.Rules { if err := Validate(rs.Rules[i]); err != nil { return fmt.Errorf("rule %d: %w", i+1, err) } if seen[rs.Rules[i].ID] { return fmt.Errorf("duplicate rule id %q", rs.Rules[i].ID) } seen[rs.Rules[i].ID] = true } return nil } func Validate(r Rule) error { if !validID(r.ID) || strings.TrimSpace(r.Title) == "" { return fmt.Errorf("invalid id/title") } switch r.Severity { case "info", "low", "medium", "high", "critical": default: return fmt.Errorf("invalid severity %q", r.Severity) } if r.Score < 0 || r.Score > 100 { return fmt.Errorf("score must be between 0 and 100") } switch r.Kind { case "event", "threshold", "distinct": default: return fmt.Errorf("invalid kind %q", r.Kind) } if len(r.EventCodes) == 0 && len(r.Conditions) == 0 { return fmt.Errorf("rule must constrain event_codes or conditions") } if r.WindowSeconds < 30 || r.WindowSeconds > 86400 { return fmt.Errorf("window_seconds must be between 30 and 86400") } if r.Threshold <= 0 { r.Threshold = 1 } if r.Kind == "distinct" { if _, ok := fieldExpr(r.DistinctField); !ok { return fmt.Errorf("invalid distinct_field %q", r.DistinctField) } if r.DistinctThreshold <= 0 { return fmt.Errorf("distinct_threshold must be positive") } } for _, g := range r.GroupBy { if _, ok := groupExpr(g); !ok { return fmt.Errorf("invalid group_by field %q", g) } } for _, c := range r.Conditions { if _, ok := fieldExpr(c.Field); !ok { return fmt.Errorf("invalid condition field %q", c.Field) } switch c.Operator { case "equals", "not_equals", "contains", "not_contains", "regex", "exists", "not_exists", "in": default: return fmt.Errorf("invalid operator %q", c.Operator) } } return nil } func Compile(r Rule, db, tenant string, end time.Time) (string, time.Time, error) { if err := Validate(r); err != nil { return "", time.Time{}, err } start := end.Add(-time.Duration(r.WindowSeconds) * time.Second) table := clickhouse.Ident(db) + ".events" where := []string{ "tenant_id=" + clickhouse.Q(tenant), "event_time>=" + clickhouse.Q(ts(start)), "event_time<" + clickhouse.Q(ts(end)), } if len(r.Channels) > 0 { where = append(where, "channel IN ("+quoteStrings(r.Channels)+")") } if len(r.EventCodes) > 0 { vals := make([]string, 0, len(r.EventCodes)) for _, id := range r.EventCodes { vals = append(vals, strconv.FormatUint(uint64(id), 10)) } where = append(where, "event_code IN ("+strings.Join(vals, ",")+")") } for _, c := range r.Conditions { x, err := compileCondition(c) if err != nil { return "", time.Time{}, err } where = append(where, x) } groupFields := make([]string, 0, len(r.GroupBy)) selectFields := make([]string, 0, len(r.GroupBy)+8) selected := map[string]bool{} for _, g := range r.GroupBy { expr, _ := groupExpr(g) alias := groupAlias(g) selectFields = append(selectFields, expr+" AS "+alias) groupFields = append(groupFields, expr) selected[g] = true } addContext := func(key, alias string) { if !selected[key] { selectFields = append(selectFields, "'' AS "+alias) } } addContext("host", "host_name") addContext("user", "user_name") addContext("source_ip", "source_ip") addContext("workstation", "workstation") addContext("process_path", "process_path") if selected["event_code"] { // already selected under event_code } else if len(r.EventCodes) == 1 { selectFields = append(selectFields, strconv.FormatUint(uint64(r.EventCodes[0]), 10)+" AS event_code") } else { selectFields = append(selectFields, "toUInt32(0) AS event_code") } selectFields = append(selectFields, "uniqExact(event_uid) AS cnt", "min(event_time) AS window_start", "max(event_time) AS window_end", ) if r.Kind == "distinct" { expr, _ := fieldExpr(r.DistinctField) selectFields = append(selectFields, "uniqExact("+expr+") AS distinct_cnt") } having := fmt.Sprintf("cnt >= %d", max64(1, r.Threshold)) if r.Kind == "distinct" { having += fmt.Sprintf(" AND distinct_cnt >= %d", r.DistinctThreshold) } q := "SELECT " + strings.Join(selectFields, ", ") + " FROM " + table + " WHERE " + strings.Join(where, " AND ") if len(groupFields) > 0 { q += " GROUP BY " + strings.Join(groupFields, ", ") } q += " HAVING " + having return q, start, nil } func RenderSummary(tpl string, row map[string]any) string { vals := map[string]string{ "host": value(row["host_name"]), "user": value(row["user_name"]), "source_ip": value(row["source_ip"]), "workstation": value(row["workstation"]), "process": value(row["process_path"]), "event_code": value(row["event_code"]), "count": value(row["cnt"]), "distinct": value(row["distinct_cnt"]), } out := tpl for k, v := range vals { out = strings.ReplaceAll(out, "{"+k+"}", v) } if strings.TrimSpace(out) == "" { out = "Rule matched on " + vals["host"] } return out } func fieldExpr(name string) (string, bool) { fields := map[string]string{ "host": "host_name", "host_name": "host_name", "user": "multiIf(target_user!='',target_user,user_name!='',user_name,subject_user)", "user_name": "user_name", "target_user": "target_user", "subject_user": "subject_user", "source_ip": "source_ip", "destination_ip": "destination_ip", "workstation": "workstation", "process_path": "process_path", "parent_process_path": "parent_process_path", "command_line": "command_line", "message": "message", "channel": "channel", "provider": "provider", "category": "category", "action": "action", "outcome": "outcome", "logon_type": "logon_type", "authentication_package": "authentication_package", "status_code": "status_code", "failure_reason": "failure_reason", "event_code": "event_code", } v, ok := fields[name] return v, ok } func groupExpr(name string) (string, bool) { return fieldExpr(name) } func groupAlias(name string) string { switch name { case "host", "host_name": return "host_name" case "user", "user_name", "target_user", "subject_user": return "user_name" default: return name } } func compileCondition(c Condition) (string, error) { expr, ok := fieldExpr(c.Field) if !ok { return "", fmt.Errorf("invalid field %q", c.Field) } q := clickhouse.Q(c.Value) switch c.Operator { case "equals": return expr + "=" + q, nil case "not_equals": return expr + "!=" + q, nil case "contains": return "positionCaseInsensitiveUTF8(" + expr + "," + q + ")>0", nil case "not_contains": return "positionCaseInsensitiveUTF8(" + expr + "," + q + ")=0", nil case "regex": return "match(" + expr + "," + q + ")", nil case "exists": return expr + "!=''", nil case "not_exists": return expr + "=''", nil case "in": parts := strings.Split(c.Value, ",") vals := make([]string, 0, len(parts)) for _, p := range parts { if x := strings.TrimSpace(p); x != "" { vals = append(vals, clickhouse.Q(x)) } } if len(vals) == 0 { return "", fmt.Errorf("empty in condition") } return expr + " IN (" + strings.Join(vals, ",") + ")", nil default: return "", fmt.Errorf("unsupported operator %q", c.Operator) } } func quoteStrings(v []string) string { out := make([]string, 0, len(v)) for _, x := range v { out = append(out, clickhouse.Q(x)) } return strings.Join(out, ",") } func validID(s string) bool { if s == "" || len(s) > 128 { return false } for _, r := range s { if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' || r == '_' || r == '.' { continue } return false } return true } func ts(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000") } func value(v any) string { if v == nil { return "" } return fmt.Sprint(v) } func max64(a, b int64) int64 { if a > b { return a } return b }