298 lines
9.4 KiB
Go
298 lines
9.4 KiB
Go
package normalize
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"io"
|
|
"net"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"example.com/siem-greenfield/internal/contracts"
|
|
)
|
|
|
|
type Parsed struct {
|
|
Computer, Provider, TargetUser, TargetDomain, SubjectUser, SubjectDomain, Workstation, SrcIP, SrcPort, DstIP, DstPort, LogonType, ProcessName, ParentProcessName, CommandLine, AuthPackage, LogonProcess, Status, SubStatus, FailureReason string
|
|
Attributes map[string]string
|
|
}
|
|
|
|
func Event(env contracts.IngestEnvelope, p contracts.LogPayload, rawKey string, partition int, offset int64, idx int) contracts.CanonicalEvent {
|
|
parsed := parseXML(p.Message)
|
|
if p.Metadata != nil {
|
|
overlay(&parsed, *p.Metadata)
|
|
}
|
|
host := first(parsed.Computer, p.Hostname)
|
|
provider := first(parsed.Provider, p.Source)
|
|
cat, action, outcome, sev := classify(p.Channel, p.EventID)
|
|
user := first(parsed.TargetUser, parsed.SubjectUser)
|
|
msg := strings.TrimSpace(p.Message)
|
|
if strings.HasPrefix(msg, "<") {
|
|
msg = ""
|
|
}
|
|
if len(msg) > 4096 {
|
|
msg = msg[:4096]
|
|
}
|
|
attrs := parsed.Attributes
|
|
if attrs == nil {
|
|
attrs = map[string]string{}
|
|
}
|
|
eventUID := env.BatchUID
|
|
if eventUID == "" {
|
|
eventUID = strconv.Itoa(partition) + ":" + strconv.FormatInt(offset, 10)
|
|
}
|
|
eventUID += ":" + strconv.Itoa(idx)
|
|
return contracts.CanonicalEvent{
|
|
EventUID: eventUID, QueuePartition: int32(partition), QueueOffset: offset,
|
|
TenantID: env.TenantID, EventTime: p.Time.UTC().Format("2006-01-02 15:04:05.000"), IngestTime: env.ReceivedAt.UTC().Format("2006-01-02 15:04:05.000"), AgentID: env.AgentID, HostName: host, SourceType: p.Source, Channel: p.Channel, Provider: provider, EventCode: p.EventID, Category: cat, Action: action, Outcome: outcome, Severity: sev,
|
|
UserName: cleanUser(user), UserDomain: first(parsed.TargetDomain, parsed.SubjectDomain), SubjectUser: cleanUser(parsed.SubjectUser), SubjectDomain: parsed.SubjectDomain, TargetUser: cleanUser(parsed.TargetUser), TargetDomain: parsed.TargetDomain, SourceIP: normalizeIP(parsed.SrcIP), SourcePort: parsePort(parsed.SrcPort), DestinationIP: normalizeIP(parsed.DstIP), DestinationPort: parsePort(parsed.DstPort), Workstation: parsed.Workstation, LogonType: parsed.LogonType, AuthenticationPackage: parsed.AuthPackage, LogonProcess: parsed.LogonProcess, StatusCode: parsed.Status, SubStatusCode: parsed.SubStatus, FailureReason: parsed.FailureReason, ProcessPath: parsed.ProcessName, ParentProcessPath: parsed.ParentProcessName, CommandLine: truncate(parsed.CommandLine, 8192), Message: msg, Attributes: attrs, RawObjectKey: rawKey, RawIndex: uint32(idx), PayloadHash: fingerprint(p), SchemaVersion: 1, ParserVersion: 1, IngestDelayMS: env.ReceivedAt.Sub(p.Time).Milliseconds(),
|
|
}
|
|
}
|
|
|
|
func overlay(p *Parsed, m contracts.EventMetadataPayload) {
|
|
p.Computer = first(m.Computer, p.Computer)
|
|
p.Provider = first(m.ProviderName, p.Provider)
|
|
p.TargetUser = first(m.TargetUser, p.TargetUser)
|
|
p.TargetDomain = first(m.TargetDomain, p.TargetDomain)
|
|
p.SubjectUser = first(m.SubjectUser, p.SubjectUser)
|
|
p.SubjectDomain = first(m.SubjectDomain, p.SubjectDomain)
|
|
p.Workstation = first(m.Workstation, m.Device, p.Workstation)
|
|
p.SrcIP = first(m.SrcIP, p.SrcIP)
|
|
p.SrcPort = first(m.SrcPort, p.SrcPort)
|
|
p.DstIP = first(m.DstIP, p.DstIP)
|
|
p.DstPort = first(m.DstPort, p.DstPort)
|
|
p.LogonType = first(m.LogonType, p.LogonType)
|
|
p.ProcessName = first(m.ProcessName, p.ProcessName)
|
|
p.ParentProcessName = first(m.ParentProcessName, p.ParentProcessName)
|
|
p.CommandLine = first(m.CommandLine, p.CommandLine)
|
|
p.AuthPackage = first(m.AuthenticationPackage, p.AuthPackage)
|
|
p.LogonProcess = first(m.LogonProcess, p.LogonProcess)
|
|
p.Status = first(m.StatusText, p.Status)
|
|
p.SubStatus = first(m.SubStatusText, p.SubStatus)
|
|
p.FailureReason = first(m.FailureReason, p.FailureReason)
|
|
}
|
|
|
|
func parseXML(s string) Parsed {
|
|
out := Parsed{Attributes: map[string]string{}}
|
|
if !strings.HasPrefix(strings.TrimSpace(s), "<") {
|
|
return out
|
|
}
|
|
dec := xml.NewDecoder(strings.NewReader(s))
|
|
var path []string
|
|
dataName := ""
|
|
for {
|
|
tok, e := dec.Token()
|
|
if e == io.EOF {
|
|
break
|
|
}
|
|
if e != nil {
|
|
return out
|
|
}
|
|
switch t := tok.(type) {
|
|
case xml.StartElement:
|
|
path = append(path, t.Name.Local)
|
|
if t.Name.Local == "Provider" {
|
|
for _, a := range t.Attr {
|
|
if a.Name.Local == "Name" {
|
|
out.Provider = strings.TrimSpace(a.Value)
|
|
}
|
|
}
|
|
}
|
|
if t.Name.Local == "Data" {
|
|
dataName = ""
|
|
for _, a := range t.Attr {
|
|
if a.Name.Local == "Name" {
|
|
dataName = strings.TrimSpace(a.Value)
|
|
}
|
|
}
|
|
}
|
|
case xml.EndElement:
|
|
if len(path) > 0 {
|
|
path = path[:len(path)-1]
|
|
}
|
|
if t.Name.Local == "Data" {
|
|
dataName = ""
|
|
}
|
|
case xml.CharData:
|
|
v := strings.TrimSpace(string(t))
|
|
if v == "" {
|
|
continue
|
|
}
|
|
if ends(path, "System", "Computer") {
|
|
out.Computer = v
|
|
continue
|
|
}
|
|
if dataName != "" {
|
|
if !promotedField(dataName) && len(out.Attributes) < 32 {
|
|
out.Attributes[dataName] = truncate(v, 1024)
|
|
}
|
|
switch dataName {
|
|
case "TargetUserName":
|
|
out.TargetUser = v
|
|
case "TargetDomainName":
|
|
out.TargetDomain = v
|
|
case "SubjectUserName":
|
|
out.SubjectUser = v
|
|
case "SubjectDomainName":
|
|
out.SubjectDomain = v
|
|
case "WorkstationName", "CallerComputerName":
|
|
out.Workstation = v
|
|
case "IpAddress", "SourceAddress":
|
|
out.SrcIP = v
|
|
case "IpPort", "SourcePort":
|
|
out.SrcPort = v
|
|
case "DestinationAddress", "DestAddress":
|
|
out.DstIP = v
|
|
case "DestinationPort", "DestPort":
|
|
out.DstPort = v
|
|
case "LogonType":
|
|
out.LogonType = v
|
|
case "ProcessName", "NewProcessName":
|
|
out.ProcessName = v
|
|
case "ParentProcessName":
|
|
out.ParentProcessName = v
|
|
case "CommandLine", "ProcessCommandLine":
|
|
out.CommandLine = v
|
|
case "AuthenticationPackageName":
|
|
out.AuthPackage = v
|
|
case "LogonProcessName":
|
|
out.LogonProcess = v
|
|
case "Status":
|
|
out.Status = v
|
|
case "SubStatus":
|
|
out.SubStatus = v
|
|
case "FailureReason":
|
|
out.FailureReason = v
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
func promotedField(name string) bool {
|
|
switch name {
|
|
case "TargetUserName", "TargetDomainName", "SubjectUserName", "SubjectDomainName",
|
|
"WorkstationName", "CallerComputerName", "IpAddress", "SourceAddress", "IpPort", "SourcePort",
|
|
"DestinationAddress", "DestAddress", "DestinationPort", "DestPort", "LogonType",
|
|
"ProcessName", "NewProcessName", "ParentProcessName", "CommandLine", "ProcessCommandLine",
|
|
"AuthenticationPackageName", "LogonProcessName", "Status", "SubStatus", "FailureReason":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func ends(path []string, p ...string) bool {
|
|
if len(path) < len(p) {
|
|
return false
|
|
}
|
|
o := len(path) - len(p)
|
|
for i := range p {
|
|
if path[o+i] != p[i] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
func first(v ...string) string {
|
|
for _, s := range v {
|
|
if s = strings.TrimSpace(s); s != "" && s != "-" {
|
|
return s
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
func cleanUser(v string) string {
|
|
v = strings.TrimSpace(v)
|
|
if v == "-" {
|
|
return ""
|
|
}
|
|
return v
|
|
}
|
|
func normalizeIP(v string) string {
|
|
v = strings.TrimSpace(v)
|
|
if v == "" || v == "-" || v == "::1" || v == "127.0.0.1" {
|
|
return ""
|
|
}
|
|
ip := net.ParseIP(v)
|
|
if ip == nil {
|
|
return ""
|
|
}
|
|
if x := ip.To4(); x != nil {
|
|
return x.String()
|
|
}
|
|
return ip.String()
|
|
}
|
|
func parsePort(v string) uint16 {
|
|
n, e := strconv.ParseUint(strings.TrimSpace(v), 10, 16)
|
|
if e != nil {
|
|
return 0
|
|
}
|
|
return uint16(n)
|
|
}
|
|
func truncate(v string, n int) string {
|
|
if len(v) > n {
|
|
return v[:n]
|
|
}
|
|
return v
|
|
}
|
|
func fingerprint(p contracts.LogPayload) string {
|
|
b, _ := json.Marshal(p)
|
|
h := sha256.Sum256(b)
|
|
return hex.EncodeToString(h[:])
|
|
}
|
|
|
|
func classify(channel string, id uint32) (string, string, string, uint8) {
|
|
if strings.EqualFold(channel, "Security") {
|
|
switch id {
|
|
case 4624:
|
|
return "authentication", "logon", "success", 2
|
|
case 4625:
|
|
return "authentication", "logon", "failure", 3
|
|
case 4648:
|
|
return "authentication", "explicit_credentials", "unknown", 3
|
|
case 4672:
|
|
return "iam", "special_privileges", "success", 3
|
|
case 4720:
|
|
return "iam", "user_created", "success", 4
|
|
case 4726:
|
|
return "iam", "user_deleted", "success", 4
|
|
case 4728, 4732, 4756:
|
|
return "iam", "privileged_group_membership_changed", "success", 5
|
|
case 4740:
|
|
return "authentication", "account_locked", "failure", 4
|
|
case 4768:
|
|
return "authentication", "kerberos_tgt", "unknown", 2
|
|
case 4769:
|
|
return "authentication", "kerberos_service_ticket", "unknown", 2
|
|
case 4771:
|
|
return "authentication", "kerberos_preauth", "failure", 3
|
|
case 4776:
|
|
return "authentication", "credential_validation", "unknown", 2
|
|
case 1102:
|
|
return "audit", "audit_log_cleared", "success", 5
|
|
}
|
|
}
|
|
switch id {
|
|
case 7045:
|
|
return "persistence", "service_installed", "success", 5
|
|
case 1074:
|
|
return "host", "shutdown_requested", "success", 2
|
|
case 6005:
|
|
return "host", "eventlog_started", "success", 1
|
|
case 6006:
|
|
return "host", "eventlog_stopped", "success", 2
|
|
}
|
|
return "event", "observed", "unknown", 1
|
|
}
|
|
|
|
func ArchiveKey(t time.Time, agent, batchUID string, partition int, offset int64) string {
|
|
id := strings.TrimSpace(batchUID)
|
|
if id == "" {
|
|
id = "p" + strconv.Itoa(partition) + "-o" + strconv.FormatInt(offset, 10)
|
|
}
|
|
return t.UTC().Format("2006/01/02/15") + "/" + strings.ReplaceAll(agent, "/", "_") + "-" + id + ".json.gz"
|
|
}
|