Merge pull request #1757 from BenCodez/codex/optional-investigation-comment

Add opt-in issue comments to investigation workflow

.github/issue-investigation/post_comment.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Publish an explicitly selected completed report; never calls OpenAI."""
+import json
+import os
+from pathlib import Path
+import re
+import sys
+
+from run import Curl
+
+RESULTS = Path('investigation-results')
+
+
+def report_links(report, repository, sha, artifacts_url):
+    # The hosted sandbox paths do not exist for issue readers. Pin source links.
+    def source(match):
+        path = match.group(1)
+        line = re.search(r':([0-9]+)$', path)
+        if line:
+            path = path[:line.start()] + '#L' + line.group(1)
+        return '](' + f'https://github.com/{repository}/blob/{sha}/' + path + ')'
+    report = re.sub(r'\]\(/workspace/repos/VotingPlugin/([^\s)]+)\)', source, report)
+    return re.sub(r'\]\(/workspace/outputs/[^\s)]+\)', '](' + artifacts_url + ')', report)
+
+
+def comment_body(report, repository, sha, run_id):
+    run_url = f'https://github.com/{repository}/actions/runs/{run_id}'
+    report = report_links(report, repository, sha, run_url + '#artifacts')
+    # Bound UTF-8 bytes as well as characters, leaving room for the heading/links.
+    encoded = report.encode('utf-8')
+    if len(encoded) > 55000:
+        report = encoded[:55000].decode('utf-8', errors='ignore') + '\n\n[Report shortened; download the complete report from the workflow artifacts.]'
+    return (f'<!-- votingplugin-issue-investigation:{run_id} -->\n'
+            '## Automated issue investigation\n\n'
+            f'[Workflow run and complete artifacts]({run_url}) · Checkout `{sha}`\n\n'
+            'This is an automated analysis, not confirmation that every check passed. '
+            'Verification results and limitations are included below.\n\n' + report.strip())
+
+
+def publish(api, repository, number, sha, run_id):
+    turn = json.loads((RESULTS / 'turn.json').read_text(encoding='utf-8'))
+    if turn.get('status') != 'completed':
+        raise RuntimeError('Refusing to post an incomplete investigation')
+    report = (RESULTS / 'report.md').read_text(encoding='utf-8')
+    if not report.strip():
+        raise RuntimeError('Refusing to post an empty report')
+    issue_path = f'/repos/{repository}/issues/{number}'
+    issue = api.request('GET', issue_path)
+    if 'pull_request' in issue:
+        raise RuntimeError('Choose an issue, not a pull request')
+    marker = f'<!-- votingplugin-issue-investigation:{run_id} -->\n'
+    # Rerunning the posting job/run must not duplicate an accepted comment,
+    # including when a prior POST succeeded but its response was lost.
+    for page in range(1, 101):
+        batch = api.request('GET', issue_path + f'/comments?per_page=100&page={page}')
+        for comment in batch:
+            if (comment.get('user', {}).get('login') == 'github-actions[bot]'
+                    and comment.get('body', '').startswith(marker)):
+                print('This workflow run already posted a report; no duplicate comment created.')
+                return
+        if len(batch) < 100:
+            break
+    else:
+        raise RuntimeError('Comment pagination limit reached; refusing to risk a duplicate')
+    api.request('POST', issue_path + '/comments', {'body': comment_body(report, repository, sha, run_id)})
+    print('Posted the completed report to issue #' + number + '.')
+
+
+def main():
+    # A second explicit gate protects against accidental direct invocation.
+    if os.environ.get('POST_COMMENT') != 'true':
+        print('Issue posting is disabled.')
+        return 0
+    try:
+        repository = os.environ['GITHUB_REPOSITORY']
+        number = os.environ['ISSUE_NUMBER']
+        sha = os.environ['GITHUB_SHA']
+        run_id = os.environ['GITHUB_RUN_ID']
+        if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repository):
+            raise RuntimeError('Invalid repository')
+        if not re.fullmatch(r'[1-9][0-9]{0,8}', number):
+            raise RuntimeError('Invalid issue number')
+        if not re.fullmatch(r'[0-9a-f]{40}', sha):
+            raise RuntimeError('Invalid checkout SHA')
+        if not re.fullmatch(r'[1-9][0-9]{0,19}', run_id):
+            raise RuntimeError('Invalid workflow run ID')
+        api = Curl('https://api.github.com', os.environ['GH_TOKEN'])
+        publish(api, repository, number, sha, run_id)
+        return 0
+    except Exception as error:
+        message = str(error) if isinstance(error, RuntimeError) else type(error).__name__
+        print('Could not post the report: ' + message, file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

.github/issue-investigation/tests/test_post_comment.py

@@ -0,0 +1,119 @@
+import importlib.util
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+with patch.object(sys, 'path', [str(ROOT), *sys.path]):
+    SPEC = importlib.util.spec_from_file_location('post_comment', ROOT / 'post_comment.py')
+    post = importlib.util.module_from_spec(SPEC)
+    SPEC.loader.exec_module(post)
+
+
+class PostingTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.folder = Path(self.temp.name)
+        result_patch = patch.object(post, 'RESULTS', self.folder)
+        result_patch.start()
+        self.addCleanup(result_patch.stop)
+        (self.folder / 'turn.json').write_text(json.dumps({'status': 'completed'}))
+        (self.folder / 'report.md').write_text('Result: no defect confirmed. Tests blocked.')
+        self.calls = []
+
+    def request(self, method, path, payload=None):
+        self.calls.append((method, path, payload))
+        return [] if method == 'GET' and '/comments?' in path else {}
+
+    def publish(self):
+        with patch.object(post.Curl, 'request', side_effect=self.request):
+            post.publish(post.Curl('https://api.github.com', 'test'), 'BenCodez/VotingPlugin', '1583', 'a' * 40, '123')
+
+    def test_default_off_never_reads_credentials_or_calls_api(self):
+        for value in [None, 'false', '1', 'True']:
+            with patch.dict(os.environ, {} if value is None else {'POST_COMMENT': value}, clear=True), \
+                    patch.object(post, 'Curl') as curl:
+                self.assertEqual(post.main(), 0)
+                curl.assert_not_called()
+
+    def test_posts_actual_completed_report_to_selected_issue(self):
+        self.publish()
+        method, path, payload = self.calls[-1]
+        self.assertEqual((method, path), ('POST', '/repos/BenCodez/VotingPlugin/issues/1583/comments'))
+        self.assertTrue(payload['body'].startswith('<!-- votingplugin-issue-investigation:123 -->\n'))
+        self.assertIn('no defect confirmed. Tests blocked.', payload['body'])
+        self.assertIn('/actions/runs/123', payload['body'])
+
+    def test_existing_bot_comment_prevents_duplicate_on_rerun(self):
+        def reply(method, path, payload=None):
+            self.calls.append((method, path, payload))
+            return [{'user': {'login': 'github-actions[bot]'},
+                     'body': '<!-- votingplugin-issue-investigation:123 -->\nold report'}] if '/comments?' in path else {}
+        self.request = reply
+        self.publish()
+        self.assertFalse(any(x[0] == 'POST' for x in self.calls))
+
+    def test_user_copied_marker_does_not_suppress_post(self):
+        def reply(method, path, payload=None):
+            self.calls.append((method, path, payload))
+            return [{'user': {'login': 'reporter'},
+                     'body': '<!-- votingplugin-issue-investigation:123 -->\nfake'}] if '/comments?' in path else {}
+        self.request = reply
+        self.publish()
+        self.assertEqual(sum(x[0] == 'POST' for x in self.calls), 1)
+
+    def test_incomplete_and_empty_reports_are_not_posted(self):
+        for status, report in [('failed', 'report'), ('in_progress', 'report'), ('completed', ' ')] :
+            (self.folder / 'turn.json').write_text(json.dumps({'status': status}))
+            (self.folder / 'report.md').write_text(report)
+            with self.assertRaises(RuntimeError):
+                self.publish()
+        self.assertEqual(self.calls, [])
+
+    def test_pr_target_is_rejected(self):
+        self.request = lambda *args: {'pull_request': {}}
+        with self.assertRaisesRegex(RuntimeError, 'not a pull request'):
+            self.publish()
+
+    def test_rewrites_sandbox_source_links_and_output_links(self):
+        text = '[Source](/workspace/repos/VotingPlugin/VotingPlugin/src/Foo.java:42) [Log](/workspace/outputs/log.txt)'
+        body = post.comment_body(text, 'BenCodez/VotingPlugin', 'a' * 40, '123')
+        self.assertIn('/blob/' + 'a' * 40 + '/VotingPlugin/src/Foo.java#L42', body)
+        self.assertIn('/actions/runs/123#artifacts', body)
+        self.assertNotIn('/workspace/', body)
+
+    def test_oversized_report_is_shortened_with_full_artifact_link(self):
+        for text in ['x' * 70000, '\U0001f600' * 20000]:
+            body = post.comment_body(text, 'BenCodez/VotingPlugin', 'a' * 40, '123')
+            self.assertLess(len(body.encode('utf-8')), 65536)
+            self.assertIn('Report shortened', body)
+
+    def test_explicit_opt_in_invokes_publisher_with_selected_issue(self):
+        env = {'POST_COMMENT': 'true', 'GH_TOKEN': 'test', 'GITHUB_REPOSITORY': 'BenCodez/VotingPlugin',
+               'ISSUE_NUMBER': '1583', 'GITHUB_SHA': 'a' * 40, 'GITHUB_RUN_ID': '123'}
+        with patch.dict(os.environ, env, clear=True), patch.object(post, 'publish') as publish:
+            self.assertEqual(post.main(), 0)
+        self.assertEqual(publish.call_args.args[1:], ('BenCodez/VotingPlugin', '1583', 'a' * 40, '123'))
+
+    def test_invalid_issue_is_rejected_before_any_api_call(self):
+        env = {'POST_COMMENT': 'true', 'GH_TOKEN': 'test', 'GITHUB_REPOSITORY': 'BenCodez/VotingPlugin',
+               'ISSUE_NUMBER': '1583/comments', 'GITHUB_SHA': 'a' * 40, 'GITHUB_RUN_ID': '123'}
+        with patch.dict(os.environ, env, clear=True), patch.object(post, 'Curl') as curl:
+            self.assertEqual(post.main(), 1)
+        curl.assert_not_called()
+
+    def test_ambiguous_post_failure_is_not_retried(self):
+        def reply(method, path, payload=None):
+            self.calls.append((method, path, payload))
+            if method == 'POST':
+                raise RuntimeError('Lost response')
+            return [] if '/comments?' in path else {}
+        self.request = reply
+        with self.assertRaisesRegex(RuntimeError, 'Lost response'):
+            self.publish()
+        self.assertEqual(sum(x[0] == 'POST' for x in self.calls), 1)

.github/workflows/investigate-issue.yml

@@ -16,6 +16,11 @@ on:
         required: true
         default: '15'
         type: string
+      post_comment:
+        description: Post the completed report as a comment on the selected issue
+        required: true
+        default: false
+        type: boolean
 
 permissions:
   contents: read
@@ -30,6 +35,8 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 50
     environment: ${{ inputs.environment }}
+    outputs:
+      artifact_id: ${{ steps.upload.outputs.artifact-id }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -51,6 +58,7 @@ jobs:
           OPENAI_PROJECT_ID: ${{ vars.OPENAI_PROJECT_ID || 'proj_aRse4SXfv5N60VlakdZGzVV5' }}
         run: python3 .github/issue-investigation/run.py --cleanup
       - name: Upload report, usage, and diagnostics
+        id: upload
         if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
@@ -58,3 +66,32 @@ jobs:
           path: investigation-results/
           retention-days: 14
           if-no-files-found: warn
+
+  post-report:
+    needs: investigate
+    if: ${{ inputs.post_comment && needs.investigate.result == 'success' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Validate the producing job's artifact ID
+        env:
+          INVESTIGATION_ARTIFACT_ID: ${{ needs.investigate.outputs.artifact_id }}
+        run: '[[ "$INVESTIGATION_ARTIFACT_ID" =~ ^[0-9]+$ ]]'
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+      - name: Download the completed investigation
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          artifact-ids: ${{ needs.investigate.outputs.artifact_id }}
+          merge-multiple: true
+          path: investigation-results/
+      - name: Post report only when explicitly selected
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ISSUE_NUMBER: ${{ inputs.issue_number }}
+          POST_COMMENT: ${{ inputs.post_comment }}
+        run: python3 .github/issue-investigation/post_comment.py

docs/issue-investigation-action.md

@@ -1,10 +1,11 @@
 # Investigate an issue with GitHub Actions
 
 The **Investigate issue** workflow runs the OpenAI-hosted issue investigation agent
-through curl. It is manual-only: it never comments, opens a PR, edits the repository,
-or starts automatically for public issue submissions. The GitHub token has only
-contents/issue read permissions and stays on the Actions runner. The OpenAI key is
-also kept on the runner, outside the hosted investigation environment.
+through curl. It is manual-only and does not start automatically for public issue
+submissions. Issue commenting is **off by default**. The investigation job has only
+contents/issue read permissions. An optional separate posting job has issue-write
+permission and receives no OpenAI key. Neither GitHub token nor the OpenAI key is
+supplied to the hosted investigation environment. No job opens PRs or edits repository files.
 
 ## Setup and use
 
@@ -16,6 +17,8 @@ also kept on the runner, outside the hosted investigation environment.
 3. After this workflow is merged to the default branch, open **Actions → Investigate
    issue → Run workflow**. Enter the issue number, select the environment containing
    the key, and optionally change the runtime limit (5–30 minutes; default 15).
+   Leave **post_comment** unchecked to keep results only in artifacts. Check it to
+   post the completed final report to that issue after a successful investigation.
 4. Download the `issue-investigation-<run-id>-<attempt>` artifact from the run page.
    It is retained for 14 days. `report.md` is the final response, `usage.json` is API
    accounting, and the session/turn/items JSON provides failure diagnostics. Published
@@ -73,3 +76,24 @@ python3 -m unittest discover -s .github/issue-investigation/tests -v
 ```
 
 These checks use fake API responses and local curl stubs, not a paid agent run.
+
+## Optional issue comment
+
+The `post_comment` checkbox authorizes posting for that manual run only. A separate
+job downloads the successful investigation's artifact by the producing job's ID
+(including on posting-only retries) and posts `report.md`, labeled
+as automated analysis, with the checkout SHA and a link to the run/full artifacts.
+The agent itself never gets write access. Failed, cancelled, incomplete, or empty
+investigations are not posted. A completed report can still describe blocked or
+failing tests; those limitations remain in the comment.
+
+The comment preserves the final report, converts sandbox source links to the exact
+GitHub commit, and links output files to the run artifacts. Reports over 55,000
+UTF-8 bytes are shortened with a link to the complete artifact. This is automatic
+publication when selected; there is no additional draft-approval step.
+
+Rerunning the same workflow run skips an existing report comment from
+`github-actions[bot]` with that run ID, including when a previous POST response was
+lost. It does not edit or replace that comment. A new manual workflow run can post a
+new report. If the posting job fails, the report artifacts remain available; rerun
+only the failed posting job to retry without paying for another investigation.