This commit is contained in:
@@ -0,0 +1,297 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"example.com/siem-greenfield/internal/contracts"
|
||||
)
|
||||
|
||||
type Parsed struct {
|
||||
Computer, Provider, TargetUser, TargetDomain, SubjectUser, SubjectDomain, Workstation, SrcIP, SrcPort, DstIP, DstPort, LogonType, ProcessName, ParentProcessName, CommandLine, AuthPackage, LogonProcess, Status, SubStatus, FailureReason string
|
||||
Attributes map[string]string
|
||||
}
|
||||
|
||||
func Event(env contracts.IngestEnvelope, p contracts.LogPayload, rawKey string, partition int, offset int64, idx int) contracts.CanonicalEvent {
|
||||
parsed := parseXML(p.Message)
|
||||
if p.Metadata != nil {
|
||||
overlay(&parsed, *p.Metadata)
|
||||
}
|
||||
host := first(parsed.Computer, p.Hostname)
|
||||
provider := first(parsed.Provider, p.Source)
|
||||
cat, action, outcome, sev := classify(p.Channel, p.EventID)
|
||||
user := first(parsed.TargetUser, parsed.SubjectUser)
|
||||
msg := strings.TrimSpace(p.Message)
|
||||
if strings.HasPrefix(msg, "<") {
|
||||
msg = ""
|
||||
}
|
||||
if len(msg) > 4096 {
|
||||
msg = msg[:4096]
|
||||
}
|
||||
attrs := parsed.Attributes
|
||||
if attrs == nil {
|
||||
attrs = map[string]string{}
|
||||
}
|
||||
eventUID := env.BatchUID
|
||||
if eventUID == "" {
|
||||
eventUID = strconv.Itoa(partition) + ":" + strconv.FormatInt(offset, 10)
|
||||
}
|
||||
eventUID += ":" + strconv.Itoa(idx)
|
||||
return contracts.CanonicalEvent{
|
||||
EventUID: eventUID, QueuePartition: int32(partition), QueueOffset: offset,
|
||||
TenantID: env.TenantID, EventTime: p.Time.UTC().Format("2006-01-02 15:04:05.000"), IngestTime: env.ReceivedAt.UTC().Format("2006-01-02 15:04:05.000"), AgentID: env.AgentID, HostName: host, SourceType: p.Source, Channel: p.Channel, Provider: provider, EventCode: p.EventID, Category: cat, Action: action, Outcome: outcome, Severity: sev,
|
||||
UserName: cleanUser(user), UserDomain: first(parsed.TargetDomain, parsed.SubjectDomain), SubjectUser: cleanUser(parsed.SubjectUser), SubjectDomain: parsed.SubjectDomain, TargetUser: cleanUser(parsed.TargetUser), TargetDomain: parsed.TargetDomain, SourceIP: normalizeIP(parsed.SrcIP), SourcePort: parsePort(parsed.SrcPort), DestinationIP: normalizeIP(parsed.DstIP), DestinationPort: parsePort(parsed.DstPort), Workstation: parsed.Workstation, LogonType: parsed.LogonType, AuthenticationPackage: parsed.AuthPackage, LogonProcess: parsed.LogonProcess, StatusCode: parsed.Status, SubStatusCode: parsed.SubStatus, FailureReason: parsed.FailureReason, ProcessPath: parsed.ProcessName, ParentProcessPath: parsed.ParentProcessName, CommandLine: truncate(parsed.CommandLine, 8192), Message: msg, Attributes: attrs, RawObjectKey: rawKey, RawIndex: uint32(idx), PayloadHash: fingerprint(p), SchemaVersion: 1, ParserVersion: 1, IngestDelayMS: env.ReceivedAt.Sub(p.Time).Milliseconds(),
|
||||
}
|
||||
}
|
||||
|
||||
func overlay(p *Parsed, m contracts.EventMetadataPayload) {
|
||||
p.Computer = first(m.Computer, p.Computer)
|
||||
p.Provider = first(m.ProviderName, p.Provider)
|
||||
p.TargetUser = first(m.TargetUser, p.TargetUser)
|
||||
p.TargetDomain = first(m.TargetDomain, p.TargetDomain)
|
||||
p.SubjectUser = first(m.SubjectUser, p.SubjectUser)
|
||||
p.SubjectDomain = first(m.SubjectDomain, p.SubjectDomain)
|
||||
p.Workstation = first(m.Workstation, m.Device, p.Workstation)
|
||||
p.SrcIP = first(m.SrcIP, p.SrcIP)
|
||||
p.SrcPort = first(m.SrcPort, p.SrcPort)
|
||||
p.DstIP = first(m.DstIP, p.DstIP)
|
||||
p.DstPort = first(m.DstPort, p.DstPort)
|
||||
p.LogonType = first(m.LogonType, p.LogonType)
|
||||
p.ProcessName = first(m.ProcessName, p.ProcessName)
|
||||
p.ParentProcessName = first(m.ParentProcessName, p.ParentProcessName)
|
||||
p.CommandLine = first(m.CommandLine, p.CommandLine)
|
||||
p.AuthPackage = first(m.AuthenticationPackage, p.AuthPackage)
|
||||
p.LogonProcess = first(m.LogonProcess, p.LogonProcess)
|
||||
p.Status = first(m.StatusText, p.Status)
|
||||
p.SubStatus = first(m.SubStatusText, p.SubStatus)
|
||||
p.FailureReason = first(m.FailureReason, p.FailureReason)
|
||||
}
|
||||
|
||||
func parseXML(s string) Parsed {
|
||||
out := Parsed{Attributes: map[string]string{}}
|
||||
if !strings.HasPrefix(strings.TrimSpace(s), "<") {
|
||||
return out
|
||||
}
|
||||
dec := xml.NewDecoder(strings.NewReader(s))
|
||||
var path []string
|
||||
dataName := ""
|
||||
for {
|
||||
tok, e := dec.Token()
|
||||
if e == io.EOF {
|
||||
break
|
||||
}
|
||||
if e != nil {
|
||||
return out
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
path = append(path, t.Name.Local)
|
||||
if t.Name.Local == "Provider" {
|
||||
for _, a := range t.Attr {
|
||||
if a.Name.Local == "Name" {
|
||||
out.Provider = strings.TrimSpace(a.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if t.Name.Local == "Data" {
|
||||
dataName = ""
|
||||
for _, a := range t.Attr {
|
||||
if a.Name.Local == "Name" {
|
||||
dataName = strings.TrimSpace(a.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
case xml.EndElement:
|
||||
if len(path) > 0 {
|
||||
path = path[:len(path)-1]
|
||||
}
|
||||
if t.Name.Local == "Data" {
|
||||
dataName = ""
|
||||
}
|
||||
case xml.CharData:
|
||||
v := strings.TrimSpace(string(t))
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if ends(path, "System", "Computer") {
|
||||
out.Computer = v
|
||||
continue
|
||||
}
|
||||
if dataName != "" {
|
||||
if !promotedField(dataName) && len(out.Attributes) < 32 {
|
||||
out.Attributes[dataName] = truncate(v, 1024)
|
||||
}
|
||||
switch dataName {
|
||||
case "TargetUserName":
|
||||
out.TargetUser = v
|
||||
case "TargetDomainName":
|
||||
out.TargetDomain = v
|
||||
case "SubjectUserName":
|
||||
out.SubjectUser = v
|
||||
case "SubjectDomainName":
|
||||
out.SubjectDomain = v
|
||||
case "WorkstationName", "CallerComputerName":
|
||||
out.Workstation = v
|
||||
case "IpAddress", "SourceAddress":
|
||||
out.SrcIP = v
|
||||
case "IpPort", "SourcePort":
|
||||
out.SrcPort = v
|
||||
case "DestinationAddress", "DestAddress":
|
||||
out.DstIP = v
|
||||
case "DestinationPort", "DestPort":
|
||||
out.DstPort = v
|
||||
case "LogonType":
|
||||
out.LogonType = v
|
||||
case "ProcessName", "NewProcessName":
|
||||
out.ProcessName = v
|
||||
case "ParentProcessName":
|
||||
out.ParentProcessName = v
|
||||
case "CommandLine", "ProcessCommandLine":
|
||||
out.CommandLine = v
|
||||
case "AuthenticationPackageName":
|
||||
out.AuthPackage = v
|
||||
case "LogonProcessName":
|
||||
out.LogonProcess = v
|
||||
case "Status":
|
||||
out.Status = v
|
||||
case "SubStatus":
|
||||
out.SubStatus = v
|
||||
case "FailureReason":
|
||||
out.FailureReason = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
func promotedField(name string) bool {
|
||||
switch name {
|
||||
case "TargetUserName", "TargetDomainName", "SubjectUserName", "SubjectDomainName",
|
||||
"WorkstationName", "CallerComputerName", "IpAddress", "SourceAddress", "IpPort", "SourcePort",
|
||||
"DestinationAddress", "DestAddress", "DestinationPort", "DestPort", "LogonType",
|
||||
"ProcessName", "NewProcessName", "ParentProcessName", "CommandLine", "ProcessCommandLine",
|
||||
"AuthenticationPackageName", "LogonProcessName", "Status", "SubStatus", "FailureReason":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func ends(path []string, p ...string) bool {
|
||||
if len(path) < len(p) {
|
||||
return false
|
||||
}
|
||||
o := len(path) - len(p)
|
||||
for i := range p {
|
||||
if path[o+i] != p[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
func first(v ...string) string {
|
||||
for _, s := range v {
|
||||
if s = strings.TrimSpace(s); s != "" && s != "-" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func cleanUser(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "-" {
|
||||
return ""
|
||||
}
|
||||
return v
|
||||
}
|
||||
func normalizeIP(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" || v == "-" || v == "::1" || v == "127.0.0.1" {
|
||||
return ""
|
||||
}
|
||||
ip := net.ParseIP(v)
|
||||
if ip == nil {
|
||||
return ""
|
||||
}
|
||||
if x := ip.To4(); x != nil {
|
||||
return x.String()
|
||||
}
|
||||
return ip.String()
|
||||
}
|
||||
func parsePort(v string) uint16 {
|
||||
n, e := strconv.ParseUint(strings.TrimSpace(v), 10, 16)
|
||||
if e != nil {
|
||||
return 0
|
||||
}
|
||||
return uint16(n)
|
||||
}
|
||||
func truncate(v string, n int) string {
|
||||
if len(v) > n {
|
||||
return v[:n]
|
||||
}
|
||||
return v
|
||||
}
|
||||
func fingerprint(p contracts.LogPayload) string {
|
||||
b, _ := json.Marshal(p)
|
||||
h := sha256.Sum256(b)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func classify(channel string, id uint32) (string, string, string, uint8) {
|
||||
if strings.EqualFold(channel, "Security") {
|
||||
switch id {
|
||||
case 4624:
|
||||
return "authentication", "logon", "success", 2
|
||||
case 4625:
|
||||
return "authentication", "logon", "failure", 3
|
||||
case 4648:
|
||||
return "authentication", "explicit_credentials", "unknown", 3
|
||||
case 4672:
|
||||
return "iam", "special_privileges", "success", 3
|
||||
case 4720:
|
||||
return "iam", "user_created", "success", 4
|
||||
case 4726:
|
||||
return "iam", "user_deleted", "success", 4
|
||||
case 4728, 4732, 4756:
|
||||
return "iam", "privileged_group_membership_changed", "success", 5
|
||||
case 4740:
|
||||
return "authentication", "account_locked", "failure", 4
|
||||
case 4768:
|
||||
return "authentication", "kerberos_tgt", "unknown", 2
|
||||
case 4769:
|
||||
return "authentication", "kerberos_service_ticket", "unknown", 2
|
||||
case 4771:
|
||||
return "authentication", "kerberos_preauth", "failure", 3
|
||||
case 4776:
|
||||
return "authentication", "credential_validation", "unknown", 2
|
||||
case 1102:
|
||||
return "audit", "audit_log_cleared", "success", 5
|
||||
}
|
||||
}
|
||||
switch id {
|
||||
case 7045:
|
||||
return "persistence", "service_installed", "success", 5
|
||||
case 1074:
|
||||
return "host", "shutdown_requested", "success", 2
|
||||
case 6005:
|
||||
return "host", "eventlog_started", "success", 1
|
||||
case 6006:
|
||||
return "host", "eventlog_stopped", "success", 2
|
||||
}
|
||||
return "event", "observed", "unknown", 1
|
||||
}
|
||||
|
||||
func ArchiveKey(t time.Time, agent, batchUID string, partition int, offset int64) string {
|
||||
id := strings.TrimSpace(batchUID)
|
||||
if id == "" {
|
||||
id = "p" + strconv.Itoa(partition) + "-o" + strconv.FormatInt(offset, 10)
|
||||
}
|
||||
return t.UTC().Format("2006/01/02/15") + "/" + strings.ReplaceAll(agent, "/", "_") + "-" + id + ".json.gz"
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"example.com/siem-greenfield/internal/contracts"
|
||||
)
|
||||
|
||||
func TestEvent4740ExtractsCallerComputer(t *testing.T) {
|
||||
xml := `<Event><System><Provider Name="Microsoft-Windows-Security-Auditing"/><Computer>DC01.example.local</Computer></System><EventData><Data Name="TargetUserName">alice</Data><Data Name="CallerComputerName">CLIENT-42</Data></EventData></Event>`
|
||||
env := contracts.IngestEnvelope{TenantID: "default", AgentID: "agent-1", BatchUID: "batch-abc", ReceivedAt: time.Date(2026, 7, 23, 12, 0, 1, 0, time.UTC)}
|
||||
p := contracts.LogPayload{Hostname: "DC01", Channel: "Security", EventID: 4740, Source: "windows-agent", Time: time.Date(2026, 7, 23, 12, 0, 0, 0, time.UTC), Message: xml}
|
||||
e := Event(env, p, "2026/07/raw.json.gz", 3, 99, 4)
|
||||
if e.EventUID != "batch-abc:4" {
|
||||
t.Fatalf("event uid = %q", e.EventUID)
|
||||
}
|
||||
if e.TargetUser != "alice" || e.Workstation != "CLIENT-42" {
|
||||
t.Fatalf("unexpected normalized event: %#v", e)
|
||||
}
|
||||
if e.Action != "account_locked" || e.Severity != 4 {
|
||||
t.Fatalf("unexpected classification: action=%s severity=%d", e.Action, e.Severity)
|
||||
}
|
||||
if e.Message != "" {
|
||||
t.Fatalf("raw XML must not be stored as message")
|
||||
}
|
||||
if len(e.Attributes) != 0 {
|
||||
t.Fatalf("promoted fields must not be duplicated in attributes: %#v", e.Attributes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWMI5857IsGenericNotDetectionClass(t *testing.T) {
|
||||
cat, action, _, sev := classify("Microsoft-Windows-WMI-Activity/Operational", 5857)
|
||||
if cat != "event" || action != "observed" || sev != 1 {
|
||||
t.Fatalf("unexpected WMI classification: %s %s %d", cat, action, sev)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user