Vollständiges Redesign
release-tag / release-image (push) Failing after 5m54s

This commit is contained in:
2026-07-23 14:45:35 +02:00
parent 42db7d18f6
commit c49bfa1c34
58 changed files with 2781 additions and 18124 deletions
-476
View File
@@ -1,476 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"math"
"net/http"
"net/url"
"os"
"os/signal"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
)
const (
maxRate = 100_000
maxWorkers = 64
maxDuration = time.Hour
maxEvents = 5_000_000
)
type config struct {
Endpoint string
Hostname string
APIKey string
EnrollmentKey string
Scenario string
Rate int
BatchSize int
Workers int
Duration time.Duration
EventLimit int64
Timeout time.Duration
Confirm bool
}
type event struct {
Hostname string `json:"host"`
Channel string `json:"channel"`
EventID uint32 `json:"id"`
Source string `json:"source"`
Time time.Time `json:"ts"`
Metadata metadata `json:"meta"`
}
type metadata struct {
Computer string `json:"computer,omitempty"`
ProviderName string `json:"provider,omitempty"`
TargetUser string `json:"target_user,omitempty"`
TargetDomain string `json:"target_domain,omitempty"`
SubjectUser string `json:"subject_user,omitempty"`
Workstation string `json:"workstation,omitempty"`
Device string `json:"device,omitempty"`
SrcIP string `json:"src_ip,omitempty"`
SrcPort string `json:"src_port,omitempty"`
LogonType string `json:"logon_type,omitempty"`
ProcessName string `json:"process_name,omitempty"`
StatusText string `json:"status,omitempty"`
SubStatus string `json:"sub_status,omitempty"`
Failure string `json:"failure_reason,omitempty"`
}
type result struct {
Events int
Duration time.Duration
Status int
Err error
}
type stats struct {
requests atomic.Int64
accepted atomic.Int64
failed atomic.Int64
bytesSent atomic.Int64
mu sync.Mutex
latencies []time.Duration
statuses map[int]int64
errors map[string]int64
}
func main() {
cfg := parseFlags()
if err := validateConfig(cfg); err != nil {
fmt.Fprintln(os.Stderr, "Konfigurationsfehler:", err)
os.Exit(2)
}
runCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
produceCtx, cancelProduction := context.WithTimeout(runCtx, cfg.Duration)
defer cancelProduction()
transport := &http.Transport{
MaxIdleConns: cfg.Workers * 2,
MaxIdleConnsPerHost: cfg.Workers,
IdleConnTimeout: 30 * time.Second,
}
client := &http.Client{Transport: transport, Timeout: cfg.Timeout}
defer transport.CloseIdleConnections()
jobs := make(chan []event, cfg.Workers*2)
results := make(chan result, cfg.Workers*2)
st := &stats{statuses: make(map[int]int64), errors: make(map[string]int64)}
var workers sync.WaitGroup
for i := 0; i < cfg.Workers; i++ {
workers.Add(1)
go func() {
defer workers.Done()
for batch := range jobs {
results <- sendBatch(runCtx, client, cfg, batch)
}
}()
}
collectorDone := make(chan struct{})
go func() {
defer close(collectorDone)
for r := range results {
st.requests.Add(1)
st.mu.Lock()
st.latencies = append(st.latencies, r.Duration)
if r.Status != 0 {
st.statuses[r.Status]++
}
if r.Err != nil {
st.failed.Add(int64(r.Events))
st.errors[shortError(r.Err)]++
} else if r.Status >= 200 && r.Status < 300 {
st.accepted.Add(int64(r.Events))
} else {
st.failed.Add(int64(r.Events))
}
st.mu.Unlock()
}
}()
fmt.Printf("SIEM-Stresstest startet: endpoint=%s host=%s scenario=%s rate=%d/s batch=%d workers=%d duration=%s limit=%d\n",
cfg.Endpoint, cfg.Hostname, cfg.Scenario, cfg.Rate, cfg.BatchSize, cfg.Workers, cfg.Duration, cfg.EventLimit)
started := time.Now()
produced := produce(produceCtx, cfg, jobs, st)
close(jobs)
workers.Wait()
close(results)
<-collectorDone
printReport(st, produced, time.Since(started))
if st.failed.Load() > 0 {
os.Exit(1)
}
}
func parseFlags() config {
var cfg config
flag.StringVar(&cfg.Endpoint, "url", getenv("SIEM_STRESS_URL", "http://127.0.0.1:8090/ingest"), "vollständige Ingest-URL")
flag.StringVar(&cfg.Hostname, "host", getenv("SIEM_STRESS_HOST", "SIEM-STRESS-01"), "Test-Hostname; alle Events eines Batches verwenden diesen Host")
flag.StringVar(&cfg.APIKey, "api-key", os.Getenv("SIEM_STRESS_API_KEY"), "API-Key des Test-Agenten")
flag.StringVar(&cfg.EnrollmentKey, "enrollment-key", os.Getenv("SIEM_STRESS_ENROLLMENT_KEY"), "Enrollment-Key für die erste Registrierung")
flag.StringVar(&cfg.Scenario, "scenario", getenv("SIEM_STRESS_SCENARIO", "mixed"), "mixed, failed-logon, lockout, normal oder catalog")
flag.IntVar(&cfg.Rate, "rate", getenvInt("SIEM_STRESS_RATE", 200), "Zielrate in Events pro Sekunde")
flag.IntVar(&cfg.BatchSize, "batch", getenvInt("SIEM_STRESS_BATCH", 100), "Events pro HTTP-Request, maximal 1000")
flag.IntVar(&cfg.Workers, "workers", getenvInt("SIEM_STRESS_WORKERS", 4), "parallele HTTP-Worker")
flag.DurationVar(&cfg.Duration, "duration", getenvDuration("SIEM_STRESS_DURATION", 30*time.Second), "maximale Laufzeit")
flag.Int64Var(&cfg.EventLimit, "max-events", getenvInt64("SIEM_STRESS_MAX_EVENTS", 100_000), "zusätzliche harte Obergrenze der erzeugten Events")
flag.DurationVar(&cfg.Timeout, "timeout", getenvDuration("SIEM_STRESS_TIMEOUT", 20*time.Second), "HTTP-Timeout pro Request")
flag.BoolVar(&cfg.Confirm, "confirm-load-test", getenvBool("SIEM_STRESS_CONFIRM", false), "bestätigt, dass das Zielsystem für diesen Lasttest autorisiert ist")
flag.Parse()
return cfg
}
func validateConfig(cfg config) error {
if !cfg.Confirm {
return errors.New("--confirm-load-test fehlt")
}
u, err := url.Parse(cfg.Endpoint)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return errors.New("--url muss eine gültige HTTP(S)-Ingest-URL sein")
}
if strings.TrimSpace(cfg.Hostname) == "" || len(cfg.Hostname) > 191 {
return errors.New("--host ist leer oder länger als 191 Zeichen")
}
if strings.TrimSpace(cfg.APIKey) == "" {
return errors.New("--api-key ist erforderlich")
}
if cfg.Rate < 1 || cfg.Rate > maxRate {
return fmt.Errorf("--rate muss zwischen 1 und %d liegen", maxRate)
}
if cfg.BatchSize < 1 || cfg.BatchSize > 1000 {
return errors.New("--batch muss zwischen 1 und 1000 liegen")
}
if cfg.Workers < 1 || cfg.Workers > maxWorkers {
return fmt.Errorf("--workers muss zwischen 1 und %d liegen", maxWorkers)
}
if cfg.Duration <= 0 || cfg.Duration > maxDuration {
return fmt.Errorf("--duration muss größer 0 und höchstens %s sein", maxDuration)
}
if cfg.EventLimit < 1 || cfg.EventLimit > maxEvents {
return fmt.Errorf("--max-events muss zwischen 1 und %d liegen", maxEvents)
}
if cfg.Timeout <= 0 || cfg.Timeout > 2*time.Minute {
return errors.New("--timeout muss größer 0 und höchstens 2m sein")
}
switch strings.ToLower(cfg.Scenario) {
case "mixed", "failed-logon", "lockout", "normal", "catalog":
default:
return errors.New("unbekanntes --scenario")
}
return nil
}
func produce(ctx context.Context, cfg config, jobs chan<- []event, st *stats) int64 {
batchInterval := time.Duration(float64(time.Second) * float64(cfg.BatchSize) / float64(cfg.Rate))
if batchInterval < time.Millisecond {
batchInterval = time.Millisecond
}
ticker := time.NewTicker(batchInterval)
defer ticker.Stop()
var produced int64
var sequence int64
for produced < cfg.EventLimit {
remaining := cfg.EventLimit - produced
n := cfg.BatchSize
if int64(n) > remaining {
n = int(remaining)
}
batch := make([]event, n)
now := time.Now().UTC()
for i := range batch {
sequence++
batch[i] = generateEvent(cfg, sequence, now.Add(time.Duration(i)*time.Microsecond))
}
encoded, _ := json.Marshal(batch)
st.bytesSent.Add(int64(len(encoded)))
select {
case jobs <- batch:
produced += int64(n)
case <-ctx.Done():
return produced
}
select {
case <-ticker.C:
case <-ctx.Done():
return produced
}
}
return produced
}
func generateEvent(cfg config, seq int64, ts time.Time) event {
base := event{
Hostname: cfg.Hostname,
Source: "SIEM-Stress-Agent",
Time: ts,
Metadata: metadata{
Computer: cfg.Hostname,
ProviderName: "SIEM-Stress-Agent",
TargetDomain: "STRESS",
SubjectUser: "stress-agent$",
Workstation: fmt.Sprintf("STRESS-CLIENT-%03d", seq%25),
Device: fmt.Sprintf("STRESS-CLIENT-%03d", seq%25),
SrcIP: fmt.Sprintf("198.18.%d.%d", (seq/250)%250, seq%250+1),
SrcPort: strconv.FormatInt(40000+seq%20000, 10),
ProcessName: `C:\\Program Files\\SIEM-Stress-Agent\\stress.exe`,
},
}
scenario := strings.ToLower(cfg.Scenario)
if scenario == "mixed" {
switch seq % 10 {
case 0, 1, 2, 3:
scenario = "failed-logon"
case 4:
scenario = "lockout"
case 5, 6, 7:
scenario = "normal"
case 8:
base.Channel, base.EventID = "System", 1074
return base
default:
base.Channel, base.EventID = "Microsoft-Windows-WMI-Activity/Operational", 5857
return base
}
}
switch scenario {
case "failed-logon":
base.Channel, base.EventID = "Security", 4625
base.Metadata.TargetUser = fmt.Sprintf("stress-user-%03d", seq%100)
base.Metadata.LogonType = "3"
base.Metadata.StatusText = "0xC000006D"
base.Metadata.SubStatus = "0xC000006A"
base.Metadata.Failure = "Unknown user name or bad password"
case "lockout":
base.Channel, base.EventID = "Security", 4740
base.Metadata.TargetUser = fmt.Sprintf("stress-lockout-%03d", seq%20)
case "normal":
base.Channel, base.EventID = "Security", 4624
base.Metadata.TargetUser = fmt.Sprintf("stress-user-%03d", seq%100)
base.Metadata.LogonType = "3"
case "catalog":
base.Channel = "SIEM-Stress-Agent/Operational"
base.EventID = uint32(50_000 + seq%5_000)
base.Metadata.TargetUser = fmt.Sprintf("catalog-user-%03d", seq%50)
}
return base
}
func sendBatch(ctx context.Context, client *http.Client, cfg config, batch []event) result {
body, err := json.Marshal(batch)
if err != nil {
return result{Events: len(batch), Err: err}
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.Endpoint, bytes.NewReader(body))
if err != nil {
return result{Events: len(batch), Err: err}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-API-Key", cfg.APIKey)
if cfg.EnrollmentKey != "" {
req.Header.Set("X-Enrollment-Key", cfg.EnrollmentKey)
}
req.Header.Set("User-Agent", "siem-stress-agent/1.0")
started := time.Now()
resp, err := client.Do(req)
duration := time.Since(started)
if err != nil {
return result{Events: len(batch), Duration: duration, Err: err}
}
defer resp.Body.Close()
limited, readErr := io.ReadAll(io.LimitReader(resp.Body, 4096))
if readErr != nil {
return result{Events: len(batch), Duration: duration, Status: resp.StatusCode, Err: readErr}
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return result{Events: len(batch), Duration: duration, Status: resp.StatusCode, Err: fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(limited)))}
}
return result{Events: len(batch), Duration: duration, Status: resp.StatusCode}
}
func printReport(st *stats, produced int64, elapsed time.Duration) {
st.mu.Lock()
latencies := append([]time.Duration(nil), st.latencies...)
statuses := cloneMap(st.statuses)
errorsByText := cloneStringMap(st.errors)
st.mu.Unlock()
sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] })
accepted := st.accepted.Load()
failed := st.failed.Load()
throughput := float64(accepted) / math.Max(elapsed.Seconds(), 0.001)
fmt.Println("\n--- Ergebnis ---")
fmt.Printf("Laufzeit: %s\n", elapsed.Round(time.Millisecond))
fmt.Printf("Erzeugte Events: %d\n", produced)
fmt.Printf("Akzeptierte Events:%d\n", accepted)
fmt.Printf("Fehlgeschlagen: %d\n", failed)
fmt.Printf("HTTP-Requests: %d\n", st.requests.Load())
fmt.Printf("Durchsatz: %.1f Events/s\n", throughput)
fmt.Printf("JSON gesendet: %.2f MiB\n", float64(st.bytesSent.Load())/(1024*1024))
if len(latencies) > 0 {
fmt.Printf("Latenz p50/p95/p99:%s / %s / %s\n", percentile(latencies, 0.50), percentile(latencies, 0.95), percentile(latencies, 0.99))
}
fmt.Printf("HTTP-Status: %v\n", statuses)
if len(errorsByText) > 0 {
fmt.Printf("Fehler: %v\n", errorsByText)
}
}
func percentile(values []time.Duration, p float64) time.Duration {
if len(values) == 0 {
return 0
}
idx := int(math.Ceil(float64(len(values))*p)) - 1
if idx < 0 {
idx = 0
}
if idx >= len(values) {
idx = len(values) - 1
}
return values[idx].Round(time.Millisecond)
}
func shortError(err error) string {
text := strings.TrimSpace(err.Error())
if len(text) > 180 {
return text[:180] + "..."
}
return text
}
func getenv(key, fallback string) string {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
return value
}
func getenvInt(key string, fallback int) int {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
n, err := strconv.Atoi(value)
if err != nil {
fmt.Fprintf(os.Stderr, "ungültiger Integer in %s: %v\n", key, err)
os.Exit(2)
}
return n
}
func getenvInt64(key string, fallback int64) int64 {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
n, err := strconv.ParseInt(value, 10, 64)
if err != nil {
fmt.Fprintf(os.Stderr, "ungültiger Integer in %s: %v\n", key, err)
os.Exit(2)
}
return n
}
func getenvDuration(key string, fallback time.Duration) time.Duration {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
d, err := time.ParseDuration(value)
if err != nil {
fmt.Fprintf(os.Stderr, "ungültige Dauer in %s: %v\n", key, err)
os.Exit(2)
}
return d
}
func getenvBool(key string, fallback bool) bool {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
parsed, err := strconv.ParseBool(value)
if err != nil {
fmt.Fprintf(os.Stderr, "ungültiger Boolean in %s: %v\n", key, err)
os.Exit(2)
}
return parsed
}
func cloneMap(in map[int]int64) map[int]int64 {
out := make(map[int]int64, len(in))
for k, v := range in {
out[k] = v
}
return out
}
func cloneStringMap(in map[string]int64) map[string]int64 {
out := make(map[string]int64, len(in))
for k, v := range in {
out[k] = v
}
return out
}
-52
View File
@@ -1,52 +0,0 @@
package main
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestSendBatch(t *testing.T) {
var got []event
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("X-API-Key") != "test-key" {
t.Fatalf("unexpected api key")
}
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
w.WriteHeader(http.StatusAccepted)
}))
defer server.Close()
cfg := config{Endpoint: server.URL, APIKey: "test-key", Timeout: time.Second}
batch := []event{generateEvent(config{Hostname: "SIEM-STRESS-01", Scenario: "lockout"}, 1, time.Now().UTC())}
res := sendBatch(context.Background(), server.Client(), cfg, batch)
if res.Err != nil || res.Status != http.StatusAccepted {
t.Fatalf("unexpected result: %+v", res)
}
if len(got) != 1 || got[0].EventID != 4740 || got[0].Source != "SIEM-Stress-Agent" {
t.Fatalf("unexpected payload: %+v", got)
}
}
func TestValidateConfigRequiresConfirmation(t *testing.T) {
cfg := config{
Endpoint: "http://127.0.0.1:8080/ingest",
Hostname: "SIEM-STRESS-01",
APIKey: "x",
Scenario: "mixed",
Rate: 1,
BatchSize: 1,
Workers: 1,
Duration: time.Second,
EventLimit: 1,
Timeout: time.Second,
}
if err := validateConfig(cfg); err == nil {
t.Fatal("expected missing confirmation to fail")
}
}
+47
View File
@@ -0,0 +1,47 @@
package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"syscall"
"example.com/siem-greenfield/internal/api"
"example.com/siem-greenfield/internal/config"
"example.com/siem-greenfield/internal/detector"
"example.com/siem-greenfield/internal/ingress"
"example.com/siem-greenfield/internal/processor"
"example.com/siem-greenfield/internal/stress"
)
func main() {
if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, "usage: siem <ingress|processor|detector|api|stress-agent>")
os.Exit(2)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
cfg := config.Load()
var err error
switch os.Args[1] {
case "ingress":
err = ingress.Run(ctx, cfg)
case "processor":
err = processor.Run(ctx, cfg)
case "detector":
err = detector.Run(ctx, cfg)
case "api":
err = api.Run(ctx, cfg)
case "stress-agent":
err = stress.Run(ctx, cfg, os.Args[2:])
default:
err = fmt.Errorf("unknown command %q", os.Args[1])
}
if err != nil {
log.Printf("fatal: %v", err)
os.Exit(1)
}
}