import fs from "node:fs/promises"; const candidates = JSON.parse(await fs.readFile("candidates.json", "utf8")); const systemPrompt = await fs.readFile("prompts/issue-resolution-system.txt", "utf8"); const outputSchema = JSON.parse(await fs.readFile("schemas/issue-resolution-output.json", "utf8")); function isMaintainerRole(role) { return ["MEMBER", "OWNER", "COLLABORATOR"].includes(role || ""); } function preScore(candidate) { let score = 0; const hardSignals = []; const contradictions = []; for (const t of candidate.timeline) { const sourceIssue = t.source?.issue; if (t.event === "cross-referenced" && sourceIssue?.pull_request?.html_url) { hardSignals.push({ type: "merged_pr", url: sourceIssue.html_url }); score += 40; // provisional until PR merged state is verified } if (["referenced", "connected"].includes(t.event)) { score += 10; } } for (const c of candidate.comments) { const body = c.body.toLowerCase(); if ( isMaintainerRole(c.author_association) && /\b(fixed|resolved|duplicate|superseded|closing)\b/.test(body) ) { score += 25; hardSignals.push({ type: "maintainer_comment", url: c.html_url }); } if (/\b(still broken|still happening|not fixed|reproducible)\b/.test(body)) { score -= 50; contradictions.push({ type: "later_unresolved_comment", url: c.html_url }); } } return { score, hardSignals, contradictions }; } function buildUserMessage(candidate) { const { issue, comments, timeline } = candidate; const commentBlock = comments .map((c) => `[${c.author_association}] ${c.user} (${c.created_at}):\n${c.body}`) .join("\n---\n"); const timelineBlock = timeline .filter((t) => ["cross-referenced", "referenced", "connected", "closed", "reopened"].includes(t.event)) .map((t) => { let line = `${t.event} (${t.created_at})`; if (t.source?.issue?.html_url) line += ` — ${t.source.issue.html_url}`; if (t.source?.issue?.pull_request?.html_url) line += ` (PR: ${t.source.issue.pull_request.html_url})`; return line; }) .join("\n"); return [ `## Issue #${issue.number}: ${issue.title}`, `URL: ${issue.html_url}`, `Created: ${issue.created_at} | Updated: ${issue.updated_at}`, `Labels: ${issue.labels.join(", ") || "none"}`, "", "### Body", issue.body || "(empty)", "", "### Comments", commentBlock || "(none)", "", "### Timeline events", timelineBlock || "(none)", ].join("\n"); } async function callGitHubModel(candidate) { const res = await fetch("https://models.inference.ai.azure.com/chat/completions", { method: "POST", headers: { Authorization: `Bearer ${process.env.GH_TOKEN}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: "gpt-4o-mini", messages: [ { role: "system", content: systemPrompt }, { role: "user", content: buildUserMessage(candidate) }, ], response_format: { type: "json_schema", json_schema: { name: "issue_resolution", strict: true, schema: outputSchema, }, }, temperature: 0.1, }), }); if (!res.ok) { const text = await res.text(); throw new Error(`GitHub Models ${res.status}: ${text}`); } const data = await res.json(); return JSON.parse(data.choices[0].message.content); } function enforcePolicy(modelOut, pre) { const approvedReasons = new Set([ "resolved_by_merged_pr", "maintainer_confirmed_resolved", "duplicate_confirmed", "superseded_confirmed" ]); const hasHardSignal = (modelOut.hard_signals || []).some(s => ["merged_pr", "maintainer_comment", "duplicate_reference", "superseded_reference"].includes(s.type) ) || pre.hardSignals.length > 0; const hasContradiction = (modelOut.contradictions || []).length > 0 || pre.contradictions.length > 0; if ( modelOut.decision === "AUTO_CLOSE" && modelOut.confidence >= 0.97 && approvedReasons.has(modelOut.reason_code) && hasHardSignal && !hasContradiction ) { return "AUTO_CLOSE"; } if ( modelOut.decision === "MANUAL_REVIEW" || modelOut.confidence >= 0.60 || pre.score >= 25 ) { return "MANUAL_REVIEW"; } return "KEEP_OPEN"; } console.log(`Classifying ${candidates.length} candidates with gpt-4o-mini...\n`); const decisions = []; for (const candidate of candidates) { const pre = preScore(candidate); const modelOut = await callGitHubModel(candidate); const finalDecision = enforcePolicy(modelOut, pre); decisions.push({ repository: candidate.repository, issue_number: candidate.issue.number, issue_url: candidate.issue.html_url, title: candidate.issue.title, pre_score: pre.score, final_decision: finalDecision, model: modelOut }); console.log( `#${candidate.issue.number} | pre_score: ${pre.score} | model: ${modelOut.decision} @ ${modelOut.confidence} | final: ${finalDecision} | ${modelOut.reason_code}` ); } await fs.writeFile("decisions.json", JSON.stringify(decisions, null, 2)); console.log(`\nWrote ${decisions.length} decisions to decisions.json`);