240 lines
8.9 KiB
Go
240 lines
8.9 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/example/glpi-ai-agent/internal/model"
|
|
"github.com/example/glpi-ai-agent/internal/ollama"
|
|
"github.com/example/glpi-ai-agent/internal/queue"
|
|
)
|
|
|
|
func (s *Service) escalationLoop(ctx context.Context) {
|
|
s.scanEscalations(ctx)
|
|
ticker := time.NewTicker(s.cfg.EscalationScanInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
s.scanEscalations(ctx)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) scanEscalations(ctx context.Context) {
|
|
lister, ok := s.glpi.(escalationLister)
|
|
if !ok {
|
|
slog.Error("escalation scanner unavailable", "reason", "GLPI connector does not implement ListEscalationCandidates")
|
|
return
|
|
}
|
|
filter := strings.TrimSpace(s.cfg.GLPIEscalationFilter)
|
|
if filter == "" {
|
|
filter = s.cfg.GLPITicketFilter
|
|
}
|
|
tickets, err := lister.ListEscalationCandidates(ctx, s.cfg.GLPIEscalationLimit, filter)
|
|
if err != nil {
|
|
s.metrics.Errors.Add(1)
|
|
slog.Error("GLPI escalation scan failed", "error", err)
|
|
return
|
|
}
|
|
now := time.Now()
|
|
for _, ticket := range tickets {
|
|
created, ok := parseGLPITime(ticket.DateCreation)
|
|
if !ok || now.Sub(created) < s.cfg.EscalationMinAge {
|
|
continue
|
|
}
|
|
if s.q.EnqueueWork(queue.WorkItem{TicketID: ticket.ID, Trigger: "scheduled_escalation", Priority: queue.PriorityScheduled}) {
|
|
s.metrics.QueueDepth.Store(int64(s.q.Len()))
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) processEscalation(ctx context.Context, item queue.WorkItem) error {
|
|
id := item.TicketID
|
|
muAny, _ := s.locks.LoadOrStore(id, &sync.Mutex{})
|
|
mu := muAny.(*sync.Mutex)
|
|
mu.Lock()
|
|
defer func() {
|
|
mu.Unlock()
|
|
s.locks.Delete(id)
|
|
}()
|
|
|
|
start := time.Now()
|
|
run := model.RunRecord{RunID: newRunID(), TicketID: id, Trigger: "scheduled_escalation", StartedAt: start, DryRun: s.cfg.DryRun, Outcome: "error"}
|
|
finish := func(err error) {
|
|
run.FinishedAt = time.Now()
|
|
if err != nil {
|
|
run.Error = err.Error()
|
|
s.metrics.Errors.Add(1)
|
|
}
|
|
if e := s.state.Append(run); e != nil {
|
|
slog.Error("persist escalation run failed", "error", e)
|
|
}
|
|
}
|
|
|
|
t, err := s.glpi.GetTicket(ctx, id)
|
|
if err != nil {
|
|
run.Reason = "ticket_load_failed"
|
|
finish(err)
|
|
return err
|
|
}
|
|
run.TicketName = t.Name
|
|
run.SourceVersion = sourceVersion(t)
|
|
run.CategoryBefore = t.CategoryID
|
|
run.PriorityBefore = t.Priority
|
|
if parent, ok := s.state.LatestTicketRun(id, "scheduled_escalation"); ok {
|
|
run.CausedByRunID = parent.RunID
|
|
}
|
|
followups, err := s.glpi.GetFollowups(ctx, id)
|
|
if err != nil {
|
|
run.Reason = "followup_check_failed"
|
|
finish(err)
|
|
return err
|
|
}
|
|
contextData := model.ContextSnapshot{}
|
|
if s.context != nil && s.cfg.ContextEnabled {
|
|
contextData = s.context.Collect(ctx, t)
|
|
}
|
|
evidence := buildEscalationEvidence(s.cfg, t, followups, contextData, time.Now())
|
|
constraints := s.escalationConstraints(contextData)
|
|
started := time.Now()
|
|
analysis := newAnalysis(run, "escalation", escalationPromptVersion, map[string]any{
|
|
"ticket": t, "followups": followups, "context": contextData,
|
|
"evidence": evidence, "constraints": constraints,
|
|
}, started)
|
|
ai, ok := s.ai.(escalationAI)
|
|
if !ok {
|
|
err = fmt.Errorf("AI client does not implement escalation analysis")
|
|
finishAnalysis(&analysis, s.cfg.OllamaModel, nil, nil, "", 0, nil, model.ActionAudit{Type: "escalation_plan", Result: "skipped: escalation_ai_unavailable"}, err)
|
|
run.Analyses = append(run.Analyses, analysis)
|
|
run.Reason = "escalation_ai_unavailable"
|
|
finish(err)
|
|
return err
|
|
}
|
|
analysisBaseCtx, escalationTrace := ollama.WithTrace(ctx, s.cfg.OllamaRoutingMode)
|
|
analysisCtx := analysisBaseCtx
|
|
cancel := func() {}
|
|
if s.cfg.EscalationAnalysisTimeout > 0 {
|
|
analysisCtx, cancel = context.WithTimeout(analysisBaseCtx, s.cfg.EscalationAnalysisTimeout)
|
|
}
|
|
decision, err := ai.AnalyseEscalation(analysisCtx, t, followups, contextData, evidence, constraints)
|
|
cancel()
|
|
if err != nil {
|
|
finishAnalysis(&analysis, s.cfg.OllamaModel, nil, nil, "", 0, nil, model.ActionAudit{Type: "escalation_plan", Result: "skipped: escalation_ai_failed"}, err)
|
|
attachAnalysisTrace(&analysis, escalationTrace)
|
|
run.Analyses = append(run.Analyses, analysis)
|
|
run.Reason = "escalation_ai_failed"
|
|
finish(err)
|
|
return err
|
|
}
|
|
result := evaluateEscalation(s.cfg, s.state, t, followups, contextData, decision, time.Now())
|
|
action := model.ActionAudit{Type: "escalation_plan", Proposed: result.Accepted, DryRun: s.cfg.DryRun || !s.cfg.AutoEscalation, Before: escalationTicketState(t), Result: result.Decision}
|
|
if result.Accepted && s.cfg.AutoEscalation && !s.cfg.DryRun {
|
|
fresh, loadErr := s.glpi.GetTicket(ctx, id)
|
|
if loadErr != nil {
|
|
err = loadErr
|
|
} else if sourceVersion(fresh) != run.SourceVersion {
|
|
result.Accepted = false
|
|
result.Decision = "escalation_ticket_changed_before_write"
|
|
action.Proposed = false
|
|
action.Result = result.Decision
|
|
} else {
|
|
freshFollowups, followupErr := s.glpi.GetFollowups(ctx, id)
|
|
if followupErr != nil {
|
|
err = followupErr
|
|
result.Accepted = false
|
|
result.Decision = "escalation_prewrite_followup_check_failed"
|
|
action.Proposed = false
|
|
action.Result = result.Decision
|
|
} else {
|
|
freshContext := contextData
|
|
if s.context != nil && s.cfg.ContextEnabled {
|
|
freshContext = s.context.Collect(ctx, fresh)
|
|
}
|
|
freshResult := evaluateEscalation(s.cfg, s.state, fresh, freshFollowups, freshContext, decision, time.Now())
|
|
freshResult.Checks = append(freshResult.Checks, model.RuleCheck{Code: "escalation_prewrite_revalidated", Group: "execution", Label: "Ticket, Followups und Kontext wurden vor dem Schreiben erneut geprüft", Status: passFail(freshResult.Accepted), Actual: freshResult.Decision, Expected: "escalation_accepted", Blocking: !freshResult.Accepted})
|
|
result = freshResult
|
|
contextData = freshContext
|
|
if !result.Accepted {
|
|
action.Proposed = false
|
|
action.Result = result.Decision
|
|
} else {
|
|
action, err = s.executeEscalationPlan(ctx, fresh, decision, result, contextData)
|
|
}
|
|
}
|
|
}
|
|
} else if result.Accepted {
|
|
action, err = s.executeEscalationPlan(ctx, t, decision, result, contextData)
|
|
}
|
|
finishAnalysis(&analysis, s.cfg.OllamaModel, result, decision.ReasonCodes, decision.Reason, decision.Confidence, result.Checks, action, err)
|
|
attachAnalysisTrace(&analysis, escalationTrace)
|
|
run.Analyses = append(run.Analyses, analysis)
|
|
run.Reason = decision.Reason
|
|
run.AIReason = decision.Reason
|
|
run.PolicyReason = result.Decision
|
|
run.Outcome = "processed"
|
|
s.metrics.EscalationRuns.Add(1)
|
|
finish(err)
|
|
return err
|
|
}
|
|
|
|
func (s *Service) escalationConstraints(contextData model.ContextSnapshot) model.EscalationConstraints {
|
|
actions := make([]string, 0, len(s.cfg.EscalationAllowedActions))
|
|
targets := make([]string, 0, 8)
|
|
for _, raw := range s.cfg.EscalationAllowedActions {
|
|
action := strings.ToLower(strings.TrimSpace(raw))
|
|
switch action {
|
|
case "none", "raise_priority":
|
|
actions = append(actions, action)
|
|
case "assign_second_level":
|
|
if s.cfg.EscalationSecondLevelGroupID > 0 {
|
|
actions = append(actions, action)
|
|
targets = append(targets, fmt.Sprintf("second_level_group:%d", s.cfg.EscalationSecondLevelGroupID))
|
|
}
|
|
case "assign_security_team":
|
|
if s.cfg.EscalationSecurityGroupID > 0 {
|
|
actions = append(actions, action)
|
|
targets = append(targets, fmt.Sprintf("security_group:%d", s.cfg.EscalationSecurityGroupID))
|
|
}
|
|
case "notify_service_owner":
|
|
if s.cfg.EscalationServiceOwnerGroupID > 0 || s.cfg.EscalationServiceOwnerUserID > 0 || s.cfg.EscalationWebhookURL != "" {
|
|
actions = append(actions, action)
|
|
targets = append(targets, "service_owner:"+actorTarget(s.cfg.EscalationServiceOwnerGroupID, s.cfg.EscalationServiceOwnerUserID, s.cfg.EscalationWebhookURL != ""))
|
|
}
|
|
case "link_major_incident":
|
|
if incident, ok := selectMajorIncident(contextData, s.cfg.EscalationMajorIncidentMinScore); ok && s.cfg.GLPIEscalationITILLinkPath != "" && s.cfg.GLPIEscalationITILLinkBody != "" {
|
|
actions = append(actions, action)
|
|
targets = append(targets, fmt.Sprintf("major_incident:%d", incident.ID))
|
|
}
|
|
case "request_manager_review":
|
|
if s.cfg.EscalationManagerReviewGroupID > 0 || s.cfg.EscalationManagerReviewUserID > 0 || s.cfg.EscalationWebhookURL != "" {
|
|
actions = append(actions, action)
|
|
targets = append(targets, "manager_review:"+actorTarget(s.cfg.EscalationManagerReviewGroupID, s.cfg.EscalationManagerReviewUserID, s.cfg.EscalationWebhookURL != ""))
|
|
}
|
|
}
|
|
}
|
|
if len(actions) == 0 {
|
|
actions = []string{"none"}
|
|
}
|
|
ownerLevel := s.cfg.EscalationServiceOwnerMinLevel
|
|
if ownerLevel <= 0 {
|
|
ownerLevel = 2
|
|
}
|
|
managerLevel := s.cfg.EscalationManagerReviewMinLevel
|
|
if managerLevel <= 0 {
|
|
managerLevel = 3
|
|
}
|
|
return model.EscalationConstraints{
|
|
AllowedActions: actions, AllowedReasonCodes: append([]string(nil), s.cfg.EscalationAllowedReasonCodes...),
|
|
MaxLevel: s.cfg.EscalationMaxLevel, MinimumAge: s.cfg.EscalationMinAge.String(),
|
|
ServiceOwnerMinLevel: ownerLevel, ManagerReviewMinLevel: managerLevel,
|
|
MajorIncidentMinRelevance: s.cfg.EscalationMajorIncidentMinScore, ConfiguredTargets: targets,
|
|
}
|
|
}
|