netbird/.github/issue-resolution/scripts/classify-candidates.mjs

import fs from "node:fs/promises";

const candidates = JSON.parse(await fs.readFile("candidates.json", "utf8"));
const systemPrompt = await fs.readFile("prompts/issue-resolution-system.txt", "utf8");
const outputSchema = JSON.parse(await fs.readFile("schemas/issue-resolution-output.json", "utf8"));

function isMaintainerRole(role) {
  return ["MEMBER", "OWNER", "COLLABORATOR"].includes(role || "");
}

function preScore(candidate) {
  let score = 0;
  const hardSignals = [];
  const contradictions = [];

  for (const t of candidate.timeline) {
    const sourceIssue = t.source?.issue;

    if (t.event === "cross-referenced" && sourceIssue?.pull_request?.html_url) {
      hardSignals.push({
        type: "merged_pr",
        url: sourceIssue.html_url
      });
      score += 40; // provisional until PR merged state is verified
    }

    if (["referenced", "connected"].includes(t.event)) {
      score += 10;
    }
  }

  for (const c of candidate.comments) {
    const body = c.body.toLowerCase();

    if (
      isMaintainerRole(c.author_association) &&
      /\b(fixed|resolved|duplicate|superseded|closing)\b/.test(body)
    ) {
      score += 25;
      hardSignals.push({
        type: "maintainer_comment",
        url: c.html_url
      });
    }

    if (/\b(still broken|still happening|not fixed|reproducible)\b/.test(body)) {
      score -= 50;
      contradictions.push({
        type: "later_unresolved_comment",
        url: c.html_url
      });
    }
  }

  return { score, hardSignals, contradictions };
}

function buildUserMessage(candidate) {
  const { issue, comments, timeline } = candidate;

  const commentBlock = comments
    .map((c) => `[${c.author_association}] ${c.user} (${c.created_at}):\n${c.body}`)
    .join("\n---\n");

  const timelineBlock = timeline
    .filter((t) => ["cross-referenced", "referenced", "connected", "closed", "reopened"].includes(t.event))
    .map((t) => {
      let line = `${t.event} (${t.created_at})`;
      if (t.source?.issue?.html_url) line += ` — ${t.source.issue.html_url}`;
      if (t.source?.issue?.pull_request?.html_url) line += ` (PR: ${t.source.issue.pull_request.html_url})`;
      return line;
    })
    .join("\n");

  return [
    `## Issue #${issue.number}: ${issue.title}`,
    `URL: ${issue.html_url}`,
    `Created: ${issue.created_at} | Updated: ${issue.updated_at}`,
    `Labels: ${issue.labels.join(", ") || "none"}`,
    "",
    "### Body",
    issue.body || "(empty)",
    "",
    "### Comments",
    commentBlock || "(none)",
    "",
    "### Timeline events",
    timelineBlock || "(none)",
  ].join("\n");
}

async function callGitHubModel(candidate) {
  const res = await fetch("https://models.inference.ai.azure.com/chat/completions", {
    method: "POST",
    headers: {
      Authorization: `Bearer ${process.env.GH_TOKEN}`,
      "Content-Type": "application/json",
    },
    body: JSON.stringify({
      model: "gpt-4o-mini",
      messages: [
        { role: "system", content: systemPrompt },
        { role: "user", content: buildUserMessage(candidate) },
      ],
      response_format: {
        type: "json_schema",
        json_schema: {
          name: "issue_resolution",
          strict: true,
          schema: outputSchema,
        },
      },
      temperature: 0.1,
    }),
  });

  if (!res.ok) {
    const text = await res.text();
    throw new Error(`GitHub Models ${res.status}: ${text}`);
  }

  const data = await res.json();
  return JSON.parse(data.choices[0].message.content);
}

function enforcePolicy(modelOut, pre) {
  const approvedReasons = new Set([
    "resolved_by_merged_pr",
    "maintainer_confirmed_resolved",
    "duplicate_confirmed",
    "superseded_confirmed"
  ]);

  const hasHardSignal =
    (modelOut.hard_signals || []).some(s =>
      ["merged_pr", "maintainer_comment", "duplicate_reference", "superseded_reference"].includes(s.type)
    ) || pre.hardSignals.length > 0;

  const hasContradiction =
    (modelOut.contradictions || []).length > 0 || pre.contradictions.length > 0;

  if (
    modelOut.decision === "AUTO_CLOSE" &&
    modelOut.confidence >= 0.97 &&
    approvedReasons.has(modelOut.reason_code) &&
    hasHardSignal &&
    !hasContradiction
  ) {
    return "AUTO_CLOSE";
  }

  if (
    modelOut.decision === "MANUAL_REVIEW" ||
    modelOut.confidence >= 0.60 ||
    pre.score >= 25
  ) {
    return "MANUAL_REVIEW";
  }

  return "KEEP_OPEN";
}

console.log(`Classifying ${candidates.length} candidates with gpt-4o-mini...\n`);

const decisions = [];
for (const candidate of candidates) {
  const pre = preScore(candidate);
  const modelOut = await callGitHubModel(candidate);
  const finalDecision = enforcePolicy(modelOut, pre);

  decisions.push({
    repository: candidate.repository,
    issue_number: candidate.issue.number,
    issue_url: candidate.issue.html_url,
    title: candidate.issue.title,
    pre_score: pre.score,
    final_decision: finalDecision,
    model: modelOut
  });

  console.log(
    `#${candidate.issue.number} | pre_score: ${pre.score} | model: ${modelOut.decision} @ ${modelOut.confidence} | final: ${finalDecision} | ${modelOut.reason_code}`
  );
}

await fs.writeFile("decisions.json", JSON.stringify(decisions, null, 2));
console.log(`\nWrote ${decisions.length} decisions to decisions.json`);