initial commit - workflow yaml, prompts and schemas

.github/issue-resolution/prompts/issue-resolution-system.txt

@@ -0,0 +1,26 @@
+You are a GitHub issue resolution classifier.
+
+Your job is to decide whether an open GitHub issue is:
+- AUTO_CLOSE
+- MANUAL_REVIEW
+- KEEP_OPEN
+
+Rules:
+1. AUTO_CLOSE is only allowed if there is objective, hard evidence:
+   - a merged linked PR that clearly resolves the issue, or
+   - an explicit maintainer/member/owner/collaborator comment saying the issue is fixed, resolved, duplicate, or superseded
+2. If there is any contradictory later evidence, do NOT AUTO_CLOSE.
+3. If evidence is promising but not airtight, choose MANUAL_REVIEW.
+4. If the issue still appears active or unresolved, choose KEEP_OPEN.
+5. Do not invent evidence.
+6. Output valid JSON only.
+
+Maintainer-authoritative roles:
+- MEMBER
+- OWNER
+- COLLABORATOR
+
+Important:
+- Later comments outweigh earlier ones.
+- A non-maintainer saying "fixed for me" is not enough for AUTO_CLOSE.
+- If uncertain, prefer MANUAL_REVIEW or KEEP_OPEN.

.github/issue-resolution/schemas/issue-resolution-output.json

@@ -0,0 +1,78 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "decision",
+    "reason_code",
+    "confidence",
+    "hard_signals",
+    "contradictions",
+    "summary",
+    "close_comment",
+    "manual_review_note"
+  ],
+  "properties": {
+    "decision": {
+      "type": "string",
+      "enum": ["AUTO_CLOSE", "MANUAL_REVIEW", "KEEP_OPEN"]
+    },
+    "reason_code": {
+      "type": "string",
+      "enum": [
+        "resolved_by_merged_pr",
+        "maintainer_confirmed_resolved",
+        "duplicate_confirmed",
+        "superseded_confirmed",
+        "likely_fixed_but_unconfirmed",
+        "still_open",
+        "unclear"
+      ]
+    },
+    "confidence": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1
+    },
+    "hard_signals": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["type", "url"],
+        "properties": {
+          "type": {
+            "type": "string",
+            "enum": [
+              "merged_pr",
+              "maintainer_comment",
+              "duplicate_reference",
+              "superseded_reference"
+            ]
+          },
+          "url": { "type": "string" }
+        }
+      }
+    },
+    "contradictions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["type", "url"],
+        "properties": {
+          "type": {
+            "type": "string",
+            "enum": [
+              "reporter_still_broken",
+              "later_unresolved_comment",
+              "ambiguous_pr_link",
+              "other"
+            ]
+          },
+          "url": { "type": "string" }
+        }
+      }
+    },
+    "summary": { "type": "string" },
+    "close_comment": { "type": "string" },
+    "manual_review_note": { "type": "string" }
+  }
+}

.github/issue-resolution/scripts/apply-decisions.mjs

@@ -0,0 +1,152 @@
+import fs from "node:fs/promises";
+
+const decisions = JSON.parse(await fs.readFile("decisions.json", "utf8"));
+const dryRun = String(process.env.DRY_RUN).toLowerCase() === "true";
+
+const headers = {
+  Authorization: `Bearer ${process.env.GH_TOKEN}`,
+  Accept: "application/vnd.github+json",
+  "X-GitHub-Api-Version": "2022-11-28",
+};
+
+async function rest(url, method = "GET", body) {
+  const res = await fetch(url, {
+    method,
+    headers,
+    body: body ? JSON.stringify(body) : undefined
+  });
+  if (!res.ok) throw new Error(`${res.status} ${url}: ${await res.text()}`);
+  return res.status === 204 ? null : res.json();
+}
+
+async function graphql(query, variables) {
+  const res = await fetch("https://api.github.com/graphql", {
+    method: "POST",
+    headers,
+    body: JSON.stringify({ query, variables })
+  });
+  if (!res.ok) throw new Error(`${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  if (json.errors) throw new Error(JSON.stringify(json.errors));
+  return json.data;
+}
+
+async function addLabel(owner, repo, issueNumber, labels) {
+  return rest(
+    `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/labels`,
+    "POST",
+    { labels }
+  );
+}
+
+async function addComment(owner, repo, issueNumber, body) {
+  return rest(
+    `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/comments`,
+    "POST",
+    { body }
+  );
+}
+
+async function closeIssue(owner, repo, issueNumber) {
+  return rest(
+    `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`,
+    "PATCH",
+    { state: "closed", state_reason: "completed" }
+  );
+}
+
+async function getIssueNodeId(owner, repo, issueNumber) {
+  const issue = await rest(`https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`);
+  return issue.node_id;
+}
+
+async function addToProject(issueNodeId) {
+  const mutation = `
+    mutation($projectId: ID!, $contentId: ID!) {
+      addProjectV2ItemById(input: {projectId: $projectId, contentId: $contentId}) {
+        item { id }
+      }
+    }
+  `;
+
+  const data = await graphql(mutation, {
+    projectId: process.env.PROJECT_ID,
+    contentId: issueNodeId
+  });
+
+  return data.addProjectV2ItemById.item.id;
+}
+
+async function setTextField(itemId, fieldId, value) {
+  const mutation = `
+    mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: String!) {
+      updateProjectV2ItemFieldValue(input: {
+        projectId: $projectId,
+        itemId: $itemId,
+        fieldId: $fieldId,
+        value: { text: $value }
+      }) {
+        projectV2Item { id }
+      }
+    }
+  `;
+
+  return graphql(mutation, {
+    projectId: process.env.PROJECT_ID,
+    itemId,
+    fieldId,
+    value
+  });
+}
+
+for (const d of decisions) {
+  const [owner, repo] = d.repository.split("/");
+
+  if (d.final_decision === "AUTO_CLOSE") {
+    if (dryRun) continue;
+
+    await addLabel(owner, repo, d.issue_number, ["auto-closed-resolved"]);
+    await addComment(
+      owner,
+      repo,
+      d.issue_number,
+      d.model.close_comment ||
+        "This appears resolved based on linked evidence, so we’re closing it automatically. Reply if this still reproduces and we’ll reopen."
+    );
+    await closeIssue(owner, repo, d.issue_number);
+  }
+
+  if (d.final_decision === "MANUAL_REVIEW") {
+    await addLabel(owner, repo, d.issue_number, ["resolution-candidate"]);
+
+    const issueNodeId = await getIssueNodeId(owner, repo, d.issue_number);
+    const itemId = await addToProject(issueNodeId);
+
+    if (process.env.PROJECT_CONFIDENCE_FIELD_ID) {
+      await setTextField(itemId, process.env.PROJECT_CONFIDENCE_FIELD_ID, String(d.model.confidence));
+    }
+    if (process.env.PROJECT_REASON_FIELD_ID) {
+      await setTextField(itemId, process.env.PROJECT_REASON_FIELD_ID, d.model.reason_code);
+    }
+    if (process.env.PROJECT_EVIDENCE_FIELD_ID) {
+      await setTextField(itemId, process.env.PROJECT_EVIDENCE_FIELD_ID, d.issue_url);
+    }
+    if (process.env.PROJECT_LINKED_PR_FIELD_ID) {
+      const linked = (d.model.hard_signals || []).map(x => x.url).join(", ");
+      if (linked) {
+        await setTextField(itemId, process.env.PROJECT_LINKED_PR_FIELD_ID, linked);
+      }
+    }
+    if (process.env.PROJECT_REPO_FIELD_ID) {
+      await setTextField(itemId, process.env.PROJECT_REPO_FIELD_ID, d.repository);
+    }
+
+    await addComment(
+      owner,
+      repo,
+      d.issue_number,
+      d.model.manual_review_note ||
+        "This issue looks like a possible resolution candidate, but not with enough certainty for automatic closure. Added to the review queue."
+    );
+  }
+}

.github/issue-resolution/scripts/classify-candidates.mjs

@@ -0,0 +1,125 @@
+import fs from "node:fs/promises";
+
+const candidates = JSON.parse(await fs.readFile("candidates.json", "utf8"));
+
+function isMaintainerRole(role) {
+  return ["MEMBER", "OWNER", "COLLABORATOR"].includes(role || "");
+}
+
+function preScore(candidate) {
+  let score = 0;
+  const hardSignals = [];
+  const contradictions = [];
+
+  for (const t of candidate.timeline) {
+    const sourceIssue = t.source?.issue;
+
+    if (t.event === "cross-referenced" && sourceIssue?.pull_request?.html_url) {
+      hardSignals.push({
+        type: "merged_pr",
+        url: sourceIssue.html_url
+      });
+      score += 40; // provisional until PR merged state is verified
+    }
+
+    if (["referenced", "connected"].includes(t.event)) {
+      score += 10;
+    }
+  }
+
+  for (const c of candidate.comments) {
+    const body = c.body.toLowerCase();
+
+    if (
+      isMaintainerRole(c.author_association) &&
+      /\b(fixed|resolved|duplicate|superseded|closing)\b/.test(body)
+    ) {
+      score += 25;
+      hardSignals.push({
+        type: "maintainer_comment",
+        url: c.html_url
+      });
+    }
+
+    if (/\b(still broken|still happening|not fixed|reproducible)\b/.test(body)) {
+      score -= 50;
+      contradictions.push({
+        type: "later_unresolved_comment",
+        url: c.html_url
+      });
+    }
+  }
+
+  return { score, hardSignals, contradictions };
+}
+
+async function callGitHubModel(issuePacket) {
+  // Replace this stub with the GitHub Models inference call used by your org.
+  // The workflow already has models: read permission.
+  return {
+    decision: "MANUAL_REVIEW",
+    reason_code: "likely_fixed_but_unconfirmed",
+    confidence: 0.74,
+    hard_signals: [],
+    contradictions: [],
+    summary: "Potential resolution candidate; evidence is not strong enough to close automatically.",
+    close_comment: "This appears resolved, so we’re closing it automatically. Reply if this is still reproducible.",
+    manual_review_note: "Potential resolution candidate. Please review evidence before closing."
+  };
+}
+
+function enforcePolicy(modelOut, pre) {
+  const approvedReasons = new Set([
+    "resolved_by_merged_pr",
+    "maintainer_confirmed_resolved",
+    "duplicate_confirmed",
+    "superseded_confirmed"
+  ]);
+
+  const hasHardSignal =
+    (modelOut.hard_signals || []).some(s =>
+      ["merged_pr", "maintainer_comment", "duplicate_reference", "superseded_reference"].includes(s.type)
+    ) || pre.hardSignals.length > 0;
+
+  const hasContradiction =
+    (modelOut.contradictions || []).length > 0 || pre.contradictions.length > 0;
+
+  if (
+    modelOut.decision === "AUTO_CLOSE" &&
+    modelOut.confidence >= 0.97 &&
+    approvedReasons.has(modelOut.reason_code) &&
+    hasHardSignal &&
+    !hasContradiction
+  ) {
+    return "AUTO_CLOSE";
+  }
+
+  if (
+    modelOut.decision === "MANUAL_REVIEW" ||
+    modelOut.confidence >= 0.60 ||
+    pre.score >= 25
+  ) {
+    return "MANUAL_REVIEW";
+  }
+
+  return "KEEP_OPEN";
+}
+
+const decisions = [];
+for (const candidate of candidates) {
+  const pre = preScore(candidate);
+  const modelOut = await callGitHubModel(candidate);
+  const finalDecision = enforcePolicy(modelOut, pre);
+
+  decisions.push({
+    repository: candidate.repository,
+    issue_number: candidate.issue.number,
+    issue_url: candidate.issue.html_url,
+    title: candidate.issue.title,
+    pre_score: pre.score,
+    final_decision: finalDecision,
+    model: modelOut
+  });
+}
+
+await fs.writeFile("decisions.json", JSON.stringify(decisions, null, 2));

.github/workflows/issue-resolution-triage.yml

@@ -0,0 +1,50 @@
+name: issue-resolution-triage
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "If true, do not close issues"
+        required: false
+        default: "true"
+      max_issues:
+        description: "How many issues to process"
+        required: false
+        default: "100"
+  schedule:
+    - cron: "17 2 * * *"
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: read
+  models: read
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      DRY_RUN: ${{ inputs.dry_run || 'true' }}
+      MAX_ISSUES: ${{ inputs.max_issues || '100' }}
+      REPO: ${{ github.repository }}
+      PROJECT_ID: ${{ vars.ISSUE_REVIEW_PROJECT_ID }}
+      PROJECT_STATUS_FIELD_ID: ${{ vars.PROJECT_STATUS_FIELD_ID }}
+      PROJECT_CONFIDENCE_FIELD_ID: ${{ vars.PROJECT_CONFIDENCE_FIELD_ID }}
+      PROJECT_REASON_FIELD_ID: ${{ vars.PROJECT_REASON_FIELD_ID }}
+      PROJECT_EVIDENCE_FIELD_ID: ${{ vars.PROJECT_EVIDENCE_FIELD_ID }}
+      PROJECT_LINKED_PR_FIELD_ID: ${{ vars.PROJECT_LINKED_PR_FIELD_ID }}
+      PROJECT_REPO_FIELD_ID: ${{ vars.PROJECT_REPO_FIELD_ID }}
+      PROJECT_STATUS_OPTION_NEEDS_REVIEW_ID: ${{ vars.PROJECT_STATUS_OPTION_NEEDS_REVIEW_ID }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - run: npm ci
+      - run: node scripts/fetch-candidates.mjs
+      - run: node scripts/classify-candidates.mjs
+      - run: node scripts/apply-decisions.mjs