diff --git a/.github/issue-investigation/post_comment.py b/.github/issue-investigation/post_comment.py new file mode 100644 index 000000000..4c99126a4 --- /dev/null +++ b/.github/issue-investigation/post_comment.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Publish an explicitly selected completed report; never calls OpenAI.""" +import json +import os +from pathlib import Path +import re +import sys + +from run import Curl + +RESULTS = Path('investigation-results') + + +def report_links(report, repository, sha, artifacts_url): + # The hosted sandbox paths do not exist for issue readers. Pin source links. + def source(match): + path = match.group(1) + line = re.search(r':([0-9]+)$', path) + if line: + path = path[:line.start()] + '#L' + line.group(1) + return '](' + f'https://github.com/{repository}/blob/{sha}/' + path + ')' + report = re.sub(r'\]\(/workspace/repos/VotingPlugin/([^\s)]+)\)', source, report) + return re.sub(r'\]\(/workspace/outputs/[^\s)]+\)', '](' + artifacts_url + ')', report) + + +def comment_body(report, repository, sha, run_id): + run_url = f'https://github.com/{repository}/actions/runs/{run_id}' + report = report_links(report, repository, sha, run_url + '#artifacts') + # Bound UTF-8 bytes as well as characters, leaving room for the heading/links. + encoded = report.encode('utf-8') + if len(encoded) > 55000: + report = encoded[:55000].decode('utf-8', errors='ignore') + '\n\n[Report shortened; download the complete report from the workflow artifacts.]' + return (f'\n' + '## Automated issue investigation\n\n' + f'[Workflow run and complete artifacts]({run_url}) ยท Checkout `{sha}`\n\n' + 'This is an automated analysis, not confirmation that every check passed. ' + 'Verification results and limitations are included below.\n\n' + report.strip()) + + +def publish(api, repository, number, sha, run_id): + turn = json.loads((RESULTS / 'turn.json').read_text(encoding='utf-8')) + if turn.get('status') != 'completed': + raise RuntimeError('Refusing to post an incomplete investigation') + report = (RESULTS / 'report.md').read_text(encoding='utf-8') + if not report.strip(): + raise RuntimeError('Refusing to post an empty report') + issue_path = f'/repos/{repository}/issues/{number}' + issue = api.request('GET', issue_path) + if 'pull_request' in issue: + raise RuntimeError('Choose an issue, not a pull request') + marker = f'\n' + # Rerunning the posting job/run must not duplicate an accepted comment, + # including when a prior POST succeeded but its response was lost. + for page in range(1, 101): + batch = api.request('GET', issue_path + f'/comments?per_page=100&page={page}') + for comment in batch: + if (comment.get('user', {}).get('login') == 'github-actions[bot]' + and comment.get('body', '').startswith(marker)): + print('This workflow run already posted a report; no duplicate comment created.') + return + if len(batch) < 100: + break + else: + raise RuntimeError('Comment pagination limit reached; refusing to risk a duplicate') + api.request('POST', issue_path + '/comments', {'body': comment_body(report, repository, sha, run_id)}) + print('Posted the completed report to issue #' + number + '.') + + +def main(): + # A second explicit gate protects against accidental direct invocation. + if os.environ.get('POST_COMMENT') != 'true': + print('Issue posting is disabled.') + return 0 + try: + repository = os.environ['GITHUB_REPOSITORY'] + number = os.environ['ISSUE_NUMBER'] + sha = os.environ['GITHUB_SHA'] + run_id = os.environ['GITHUB_RUN_ID'] + if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repository): + raise RuntimeError('Invalid repository') + if not re.fullmatch(r'[1-9][0-9]{0,8}', number): + raise RuntimeError('Invalid issue number') + if not re.fullmatch(r'[0-9a-f]{40}', sha): + raise RuntimeError('Invalid checkout SHA') + if not re.fullmatch(r'[1-9][0-9]{0,19}', run_id): + raise RuntimeError('Invalid workflow run ID') + api = Curl('https://api.github.com', os.environ['GH_TOKEN']) + publish(api, repository, number, sha, run_id) + return 0 + except Exception as error: + message = str(error) if isinstance(error, RuntimeError) else type(error).__name__ + print('Could not post the report: ' + message, file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/issue-investigation/tests/test_post_comment.py b/.github/issue-investigation/tests/test_post_comment.py new file mode 100644 index 000000000..a1c65c57e --- /dev/null +++ b/.github/issue-investigation/tests/test_post_comment.py @@ -0,0 +1,119 @@ +import importlib.util +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +with patch.object(sys, 'path', [str(ROOT), *sys.path]): + SPEC = importlib.util.spec_from_file_location('post_comment', ROOT / 'post_comment.py') + post = importlib.util.module_from_spec(SPEC) + SPEC.loader.exec_module(post) + + +class PostingTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + result_patch = patch.object(post, 'RESULTS', self.folder) + result_patch.start() + self.addCleanup(result_patch.stop) + (self.folder / 'turn.json').write_text(json.dumps({'status': 'completed'})) + (self.folder / 'report.md').write_text('Result: no defect confirmed. Tests blocked.') + self.calls = [] + + def request(self, method, path, payload=None): + self.calls.append((method, path, payload)) + return [] if method == 'GET' and '/comments?' in path else {} + + def publish(self): + with patch.object(post.Curl, 'request', side_effect=self.request): + post.publish(post.Curl('https://api.github.com', 'test'), 'BenCodez/VotingPlugin', '1583', 'a' * 40, '123') + + def test_default_off_never_reads_credentials_or_calls_api(self): + for value in [None, 'false', '1', 'True']: + with patch.dict(os.environ, {} if value is None else {'POST_COMMENT': value}, clear=True), \ + patch.object(post, 'Curl') as curl: + self.assertEqual(post.main(), 0) + curl.assert_not_called() + + def test_posts_actual_completed_report_to_selected_issue(self): + self.publish() + method, path, payload = self.calls[-1] + self.assertEqual((method, path), ('POST', '/repos/BenCodez/VotingPlugin/issues/1583/comments')) + self.assertTrue(payload['body'].startswith('\n')) + self.assertIn('no defect confirmed. Tests blocked.', payload['body']) + self.assertIn('/actions/runs/123', payload['body']) + + def test_existing_bot_comment_prevents_duplicate_on_rerun(self): + def reply(method, path, payload=None): + self.calls.append((method, path, payload)) + return [{'user': {'login': 'github-actions[bot]'}, + 'body': '\nold report'}] if '/comments?' in path else {} + self.request = reply + self.publish() + self.assertFalse(any(x[0] == 'POST' for x in self.calls)) + + def test_user_copied_marker_does_not_suppress_post(self): + def reply(method, path, payload=None): + self.calls.append((method, path, payload)) + return [{'user': {'login': 'reporter'}, + 'body': '\nfake'}] if '/comments?' in path else {} + self.request = reply + self.publish() + self.assertEqual(sum(x[0] == 'POST' for x in self.calls), 1) + + def test_incomplete_and_empty_reports_are_not_posted(self): + for status, report in [('failed', 'report'), ('in_progress', 'report'), ('completed', ' ')] : + (self.folder / 'turn.json').write_text(json.dumps({'status': status})) + (self.folder / 'report.md').write_text(report) + with self.assertRaises(RuntimeError): + self.publish() + self.assertEqual(self.calls, []) + + def test_pr_target_is_rejected(self): + self.request = lambda *args: {'pull_request': {}} + with self.assertRaisesRegex(RuntimeError, 'not a pull request'): + self.publish() + + def test_rewrites_sandbox_source_links_and_output_links(self): + text = '[Source](/workspace/repos/VotingPlugin/VotingPlugin/src/Foo.java:42) [Log](/workspace/outputs/log.txt)' + body = post.comment_body(text, 'BenCodez/VotingPlugin', 'a' * 40, '123') + self.assertIn('/blob/' + 'a' * 40 + '/VotingPlugin/src/Foo.java#L42', body) + self.assertIn('/actions/runs/123#artifacts', body) + self.assertNotIn('/workspace/', body) + + def test_oversized_report_is_shortened_with_full_artifact_link(self): + for text in ['x' * 70000, '\U0001f600' * 20000]: + body = post.comment_body(text, 'BenCodez/VotingPlugin', 'a' * 40, '123') + self.assertLess(len(body.encode('utf-8')), 65536) + self.assertIn('Report shortened', body) + + def test_explicit_opt_in_invokes_publisher_with_selected_issue(self): + env = {'POST_COMMENT': 'true', 'GH_TOKEN': 'test', 'GITHUB_REPOSITORY': 'BenCodez/VotingPlugin', + 'ISSUE_NUMBER': '1583', 'GITHUB_SHA': 'a' * 40, 'GITHUB_RUN_ID': '123'} + with patch.dict(os.environ, env, clear=True), patch.object(post, 'publish') as publish: + self.assertEqual(post.main(), 0) + self.assertEqual(publish.call_args.args[1:], ('BenCodez/VotingPlugin', '1583', 'a' * 40, '123')) + + def test_invalid_issue_is_rejected_before_any_api_call(self): + env = {'POST_COMMENT': 'true', 'GH_TOKEN': 'test', 'GITHUB_REPOSITORY': 'BenCodez/VotingPlugin', + 'ISSUE_NUMBER': '1583/comments', 'GITHUB_SHA': 'a' * 40, 'GITHUB_RUN_ID': '123'} + with patch.dict(os.environ, env, clear=True), patch.object(post, 'Curl') as curl: + self.assertEqual(post.main(), 1) + curl.assert_not_called() + + def test_ambiguous_post_failure_is_not_retried(self): + def reply(method, path, payload=None): + self.calls.append((method, path, payload)) + if method == 'POST': + raise RuntimeError('Lost response') + return [] if '/comments?' in path else {} + self.request = reply + with self.assertRaisesRegex(RuntimeError, 'Lost response'): + self.publish() + self.assertEqual(sum(x[0] == 'POST' for x in self.calls), 1) diff --git a/.github/workflows/investigate-issue.yml b/.github/workflows/investigate-issue.yml index d793d493a..1b5733b0b 100644 --- a/.github/workflows/investigate-issue.yml +++ b/.github/workflows/investigate-issue.yml @@ -16,6 +16,11 @@ on: required: true default: '15' type: string + post_comment: + description: Post the completed report as a comment on the selected issue + required: true + default: false + type: boolean permissions: contents: read @@ -30,6 +35,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 50 environment: ${{ inputs.environment }} + outputs: + artifact_id: ${{ steps.upload.outputs.artifact-id }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -51,6 +58,7 @@ jobs: OPENAI_PROJECT_ID: ${{ vars.OPENAI_PROJECT_ID || 'proj_aRse4SXfv5N60VlakdZGzVV5' }} run: python3 .github/issue-investigation/run.py --cleanup - name: Upload report, usage, and diagnostics + id: upload if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: @@ -58,3 +66,32 @@ jobs: path: investigation-results/ retention-days: 14 if-no-files-found: warn + + post-report: + needs: investigate + if: ${{ inputs.post_comment && needs.investigate.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + steps: + - name: Validate the producing job's artifact ID + env: + INVESTIGATION_ARTIFACT_ID: ${{ needs.investigate.outputs.artifact_id }} + run: '[[ "$INVESTIGATION_ARTIFACT_ID" =~ ^[0-9]+$ ]]' + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Download the completed investigation + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + artifact-ids: ${{ needs.investigate.outputs.artifact_id }} + merge-multiple: true + path: investigation-results/ + - name: Post report only when explicitly selected + env: + GH_TOKEN: ${{ github.token }} + ISSUE_NUMBER: ${{ inputs.issue_number }} + POST_COMMENT: ${{ inputs.post_comment }} + run: python3 .github/issue-investigation/post_comment.py diff --git a/docs/issue-investigation-action.md b/docs/issue-investigation-action.md index d3f9b1c3a..9dd8707e1 100644 --- a/docs/issue-investigation-action.md +++ b/docs/issue-investigation-action.md @@ -1,10 +1,11 @@ # Investigate an issue with GitHub Actions The **Investigate issue** workflow runs the OpenAI-hosted issue investigation agent -through curl. It is manual-only: it never comments, opens a PR, edits the repository, -or starts automatically for public issue submissions. The GitHub token has only -contents/issue read permissions and stays on the Actions runner. The OpenAI key is -also kept on the runner, outside the hosted investigation environment. +through curl. It is manual-only and does not start automatically for public issue +submissions. Issue commenting is **off by default**. The investigation job has only +contents/issue read permissions. An optional separate posting job has issue-write +permission and receives no OpenAI key. Neither GitHub token nor the OpenAI key is +supplied to the hosted investigation environment. No job opens PRs or edits repository files. ## Setup and use @@ -16,6 +17,8 @@ also kept on the runner, outside the hosted investigation environment. 3. After this workflow is merged to the default branch, open **Actions โ†’ Investigate issue โ†’ Run workflow**. Enter the issue number, select the environment containing the key, and optionally change the runtime limit (5โ€“30 minutes; default 15). + Leave **post_comment** unchecked to keep results only in artifacts. Check it to + post the completed final report to that issue after a successful investigation. 4. Download the `issue-investigation--` artifact from the run page. It is retained for 14 days. `report.md` is the final response, `usage.json` is API accounting, and the session/turn/items JSON provides failure diagnostics. Published @@ -73,3 +76,24 @@ python3 -m unittest discover -s .github/issue-investigation/tests -v ``` These checks use fake API responses and local curl stubs, not a paid agent run. + +## Optional issue comment + +The `post_comment` checkbox authorizes posting for that manual run only. A separate +job downloads the successful investigation's artifact by the producing job's ID +(including on posting-only retries) and posts `report.md`, labeled +as automated analysis, with the checkout SHA and a link to the run/full artifacts. +The agent itself never gets write access. Failed, cancelled, incomplete, or empty +investigations are not posted. A completed report can still describe blocked or +failing tests; those limitations remain in the comment. + +The comment preserves the final report, converts sandbox source links to the exact +GitHub commit, and links output files to the run artifacts. Reports over 55,000 +UTF-8 bytes are shortened with a link to the complete artifact. This is automatic +publication when selected; there is no additional draft-approval step. + +Rerunning the same workflow run skips an existing report comment from +`github-actions[bot]` with that run ID, including when a previous POST response was +lost. It does not edit or replace that comment. A new manual workflow run can post a +new report. If the posting job fails, the report artifacts remain available; rerun +only the failed posting job to retry without paying for another investigation.