Issue Management Workflow
L3
GithubMCPMark CI/CD
Build intelligent issue management automation with auto-triage, task breakdown for epics, and first-time contributor handling.
Created by Zijian Wu
2025-08-15
Issue ManagementWorkflow Automation
Model Ranking
Click on the dots to view the trajectory of each task run
Model | Run Results | Pass@4 | Pass^4 | Avg Time | Avg Turns | Input Tokens | Output Tokens | Total Tokens |
---|---|---|---|---|---|---|---|---|
claude-4-1-opus | 0 /1 | - | - | 180.3s | 1.0 | 17,147 | 160 | 17,307 |
claude-4-sonnet | 0 /4 | 371.5s | 13.3 | 706,175 | 8,383 | 714,558 | ||
deepseek-chat | 0 /4 | 382.4s | 11.8 | 285,917 | 6,322 | 292,239 | ||
gemini-2-5-pro | 0 /4 | 126.9s | 8.3 | 137,288 | 11,368 | 148,655 | ||
gpt-5 | 0 /4 | 706.0s | 15.5 | 426,461 | 40,184 | 466,645 | ||
grok-4 | 0 /4 | 30.6s | - | - | - | - | ||
k2 | 0 /4 | 199.3s | 7.8 | 169,025 | 3,876 | 172,901 | ||
o3 | 0 /4 | 110.7s | 5.3 | 77,118 | 5,105 | 82,223 | ||
qwen-3-coder | 0 /4 | 181.6s | 15.5 | 510,108 | 5,684 | 515,792 |
Task State
Instruction
Verify
Python
import sys
import os
import requests
import time
from typing import Dict, List, Optional, Tuple
from dotenv import load_dotenv
def _get_github_api(
endpoint: str, headers: Dict[str, str], owner: str, repo: str
) -> Tuple[bool, Optional[Dict]]:
"""Make a GET request to GitHub API and return (success, response)."""
url = f"https://api.github.com/repos/{owner}/{repo}/{endpoint}"
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
return True, response.json()
elif response.status_code == 404:
return False, None
else:
print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
return False, None
except Exception as e:
print(f"Exception for {endpoint}: {e}", file=sys.stderr)
return False, None
def _search_github_issues(
query: str, headers: Dict[str, str]
) -> Tuple[bool, Optional[List]]:
"""Search GitHub issues using the search API."""
url = f"https://api.github.com/search/issues?q={query}&per_page=100"
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
data = response.json()
return True, data.get("items", [])
else:
print(f"Search API error: {response.status_code}", file=sys.stderr)
return False, None
except Exception as e:
print(f"Search exception: {e}", file=sys.stderr)
return False, None
def _wait_for_workflow_completion(
headers: Dict[str, str], owner: str, repo: str, max_wait: int = 180
) -> bool:
"""Wait for GitHub Actions workflows to complete processing."""
print("⏳ Waiting for GitHub Actions workflows to complete...")
start_time = time.time()
expected_runs = 3 # We created 3 test issues
no_workflow_check_count = 0
while time.time() - start_time < max_wait:
try:
# Check workflow runs
success, response = _get_github_api(
"actions/workflows/issue-automation.yml/runs?per_page=20",
headers,
owner,
repo,
)
if success and response:
runs = response.get("workflow_runs", [])
if len(runs) >= expected_runs:
# Check status of recent runs
recent_runs = runs[:expected_runs]
running_count = 0
completed_count = 0
failed_count = 0
for run in recent_runs:
status = run["status"]
conclusion = run.get("conclusion")
if status == "completed":
completed_count += 1
if conclusion == "failure":
failed_count += 1
elif status in ["in_progress", "queued"]:
running_count += 1
print(
f" Status: {completed_count} completed, {running_count} running/queued"
)
# Wait until NO workflows are running and we have enough completed runs
if running_count == 0 and completed_count >= expected_runs:
if failed_count > 0:
print(
f"⚠️ Warning: {failed_count} workflow runs failed, but continuing verification..."
)
print(
f"✅ All workflows completed. Found {completed_count} completed runs."
)
# Additional wait to ensure all issue processing is done
print("⏳ Additional wait for issue processing to complete...")
time.sleep(5)
return True
elif len(runs) == 0:
# No workflow runs found
no_workflow_check_count += 1
if no_workflow_check_count == 1:
print(
" No workflow runs found yet, waiting 5 seconds and checking once more..."
)
time.sleep(5)
continue
elif no_workflow_check_count >= 2:
print(
"⚠️ No workflow runs detected after 2 checks. Workflow may not have been triggered."
)
print(" Continuing with verification...")
return False
else:
print(
f" Waiting for workflow runs... Found {len(runs)}, expected {expected_runs}"
)
print(f"⏳ Still waiting... ({int(time.time() - start_time)}s elapsed)")
time.sleep(5)
except Exception as e:
print(f"⚠️ Error checking workflow status: {e}")
time.sleep(5)
print(f"⚠️ Workflow completion wait timed out after {max_wait}s")
return False
def _find_issue_by_title(
title: str, headers: Dict[str, str], owner: str, repo: str
) -> Optional[Dict]:
"""Find an issue by exact title match."""
success, issues = _search_github_issues(
f'repo:{owner}/{repo} "{title}" is:issue', headers
)
if success and issues:
for issue in issues:
if issue.get("title") == title:
return issue
return None
def _check_issue_labels(
issue: Dict, expected_labels: List[str]
) -> Tuple[bool, List[str]]:
"""Check if issue has the expected labels."""
actual_labels = [label["name"] for label in issue.get("labels", [])]
missing_labels = [label for label in expected_labels if label not in actual_labels]
if missing_labels:
return False, [f"Missing labels: {missing_labels}. Found: {actual_labels}"]
return True, []
def _check_issue_milestone(
issue: Dict, expected_milestone: str
) -> Tuple[bool, List[str]]:
"""Check if issue has the expected milestone."""
milestone = issue.get("milestone")
if not milestone:
if expected_milestone:
return False, [f"No milestone found. Expected: {expected_milestone}"]
return True, []
if milestone.get("title") != expected_milestone:
return False, [
f"Wrong milestone: {milestone.get('title')}. Expected: {expected_milestone}"
]
return True, []
def _check_issue_comments(
issue_number: int,
expected_content: str,
headers: Dict[str, str],
owner: str,
repo: str,
) -> Tuple[bool, List[str]]:
"""Check if issue has a comment containing expected content."""
success, comments = _get_github_api(
f"issues/{issue_number}/comments", headers, owner, repo
)
if not success:
return False, ["Failed to get issue comments"]
if not comments:
return False, [f"No comments found. Expected comment with: {expected_content}"]
for comment in comments:
if expected_content in comment.get("body", ""):
return True, []
return False, [f"Expected content '{expected_content}' not found in comments"]
def _find_epic_sub_issues(
parent_issue_number: int, headers: Dict[str, str], owner: str, repo: str
) -> Tuple[List[Dict], List[str]]:
"""Find sub-issues created for an epic."""
# Search for each expected sub-task by exact title
expected_subtasks = [
"[SUBTASK] Epic: Redesign user dashboard interface - Task 1: Requirements Analysis",
"[SUBTASK] Epic: Redesign user dashboard interface - Task 2: Design and Architecture",
"[SUBTASK] Epic: Redesign user dashboard interface - Task 3: Implementation",
"[SUBTASK] Epic: Redesign user dashboard interface - Task 4: Testing and Documentation",
]
subtasks = []
errors = []
for expected_title in expected_subtasks:
# Search for exact title
success, issues = _search_github_issues(
f'repo:{owner}/{repo} "{expected_title}" is:issue', headers
)
if not success:
errors.append(f"Failed to search for sub-issue: {expected_title}")
continue
# Find exact match
found = False
for issue in issues:
if issue.get("title") == expected_title:
# Verify it references the parent issue
body = issue.get("body", "")
if (
f"#{parent_issue_number}" in body
or f"Related to #{parent_issue_number}" in body
):
subtasks.append(issue)
found = True
break
if not found:
errors.append(
f"Sub-issue not found or doesn't reference parent: {expected_title}"
)
return subtasks, errors
def _check_epic_checklist(
issue: Dict, subtask_numbers: List[int]
) -> Tuple[bool, List[str]]:
"""Check if epic issue has the Epic Tasks checklist with correct issue references."""
body = issue.get("body", "")
errors = []
if "## Epic Tasks" not in body:
return False, ["Epic Tasks section not found in issue body"]
# Check that all subtask issue numbers are referenced in checkbox format
for number in subtask_numbers:
# Check for checkbox format: - [ ] #number
if f"- [ ] #{number}" not in body:
errors.append(
f"Sub-issue #{number} not found in Epic Tasks checklist format (expected: '- [ ] #{number}')"
)
# Also verify the expected task names are present
expected_tasks = [
"Requirements Analysis",
"Design and Architecture",
"Implementation",
"Testing and Documentation",
]
for task in expected_tasks:
if task not in body:
errors.append(f"Task name '{task}' not found in Epic Tasks section")
if errors:
return False, errors
return True, []
def _verify_bug_issue(
headers: Dict[str, str], owner: str, repo: str
) -> Tuple[bool, List[str]]:
"""Verify the bug issue requirements."""
print("\n🐛 Verifying Bug Issue...")
errors = []
# Find bug issue
bug_issue = _find_issue_by_title(
"Bug: Login form validation not working", headers, owner, repo
)
if not bug_issue:
return False, ["Bug issue 'Bug: Login form validation not working' not found"]
issue_number = bug_issue["number"]
print(f" Found bug issue #{issue_number}")
# Check labels (including first-time-contributor since it's the first issue)
expected_labels = ["bug", "priority-high", "needs-review", "first-time-contributor"]
labels_ok, label_errors = _check_issue_labels(bug_issue, expected_labels)
if not labels_ok:
errors.extend(label_errors)
else:
print(f" ✅ Labels verified: {expected_labels}")
# Check milestone
milestone_ok, milestone_errors = _check_issue_milestone(bug_issue, "v1.0.0")
if not milestone_ok:
errors.extend(milestone_errors)
else:
print(" ✅ Milestone verified: v1.0.0")
# Check comment
comment_ok, comment_errors = _check_issue_comments(
issue_number, "Bug Report Guidelines", headers, owner, repo
)
if not comment_ok:
errors.extend(comment_errors)
else:
print(" ✅ Bug Report Guidelines comment found")
return len(errors) == 0, errors
def _verify_epic_issue(
headers: Dict[str, str], owner: str, repo: str
) -> Tuple[bool, List[str]]:
"""Verify the epic issue requirements."""
print("\n🚀 Verifying Epic Issue...")
errors = []
# Find epic issue
epic_issue = _find_issue_by_title(
"Epic: Redesign user dashboard interface", headers, owner, repo
)
if not epic_issue:
return False, ["Epic issue 'Epic: Redesign user dashboard interface' not found"]
issue_number = epic_issue["number"]
print(f" Found epic issue #{issue_number}")
# Check labels
expected_labels = ["epic", "priority-high", "needs-review"]
labels_ok, label_errors = _check_issue_labels(epic_issue, expected_labels)
if not labels_ok:
errors.extend(label_errors)
else:
print(f" ✅ Labels verified: {expected_labels}")
# Check milestone
milestone_ok, milestone_errors = _check_issue_milestone(epic_issue, "v1.0.0")
if not milestone_ok:
errors.extend(milestone_errors)
else:
print(" ✅ Milestone verified: v1.0.0")
# Check comment
comment_ok, comment_errors = _check_issue_comments(
issue_number, "Feature Request Process", headers, owner, repo
)
if not comment_ok:
errors.extend(comment_errors)
else:
print(" ✅ Feature Request Process comment found")
# Find and verify sub-issues
sub_issues, sub_errors = _find_epic_sub_issues(issue_number, headers, owner, repo)
if sub_errors:
errors.extend(sub_errors)
elif len(sub_issues) != 4:
errors.append(f"Expected 4 sub-issues, found {len(sub_issues)}")
else:
print(f" ✅ Found {len(sub_issues)} sub-issues")
# Collect sub-issue numbers for checklist verification
subtask_numbers = []
# Verify each sub-issue has correct labels and link to parent
for sub_issue in sub_issues:
sub_number = sub_issue["number"]
subtask_numbers.append(sub_number)
# Check labels
sub_labels = [label["name"] for label in sub_issue.get("labels", [])]
expected_sub_labels = ["enhancement", "needs-review"]
missing_sub_labels = [
label for label in expected_sub_labels if label not in sub_labels
]
if missing_sub_labels:
errors.append(
f"Sub-issue #{sub_number} missing labels: {missing_sub_labels}"
)
# Verify parent reference in body
sub_body = sub_issue.get("body", "")
if (
f"#{issue_number}" not in sub_body
and f"Related to #{issue_number}" not in sub_body
):
errors.append(
f"Sub-issue #{sub_number} doesn't reference parent issue #{issue_number}"
)
if not errors:
print(
" ✅ All 4 sub-tasks created with correct labels and parent references"
)
# Check Epic Tasks checklist with correct issue numbers
checklist_ok, checklist_errors = _check_epic_checklist(
epic_issue, subtask_numbers
)
if not checklist_ok:
errors.extend(checklist_errors)
else:
print(
f" ✅ Epic Tasks checklist verified with correct issue references: {subtask_numbers}"
)
return len(errors) == 0, errors
def _verify_maintenance_issue(
headers: Dict[str, str], owner: str, repo: str
) -> Tuple[bool, List[str]]:
"""Verify the maintenance issue requirements."""
print("\n🔧 Verifying Maintenance Issue...")
errors = []
# Find maintenance issue
maintenance_issue = _find_issue_by_title(
"Weekly maintenance cleanup and refactor", headers, owner, repo
)
if not maintenance_issue:
return False, [
"Maintenance issue 'Weekly maintenance cleanup and refactor' not found"
]
issue_number = maintenance_issue["number"]
print(f" Found maintenance issue #{issue_number}")
# Check labels
expected_labels = ["maintenance", "priority-medium", "needs-review"]
labels_ok, label_errors = _check_issue_labels(maintenance_issue, expected_labels)
if not labels_ok:
errors.extend(label_errors)
else:
print(f" ✅ Labels verified: {expected_labels}")
# Check NO milestone (maintenance issues shouldn't get v1.0.0)
milestone_ok, milestone_errors = _check_issue_milestone(maintenance_issue, None)
if not milestone_ok:
errors.extend(milestone_errors)
else:
print(" ✅ No milestone assigned (correct for maintenance issue)")
# Check comment
comment_ok, comment_errors = _check_issue_comments(
issue_number, "Maintenance Guidelines", headers, owner, repo
)
if not comment_ok:
errors.extend(comment_errors)
else:
print(" ✅ Maintenance Guidelines comment found")
return len(errors) == 0, errors
def verify() -> bool:
"""
Verify that the issue management workflow automation is working correctly.
"""
# Load environment variables
load_dotenv(".mcp_env")
github_token = os.environ.get("MCP_GITHUB_TOKEN")
if not github_token:
print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
return False
# Get GitHub organization
github_org = os.environ.get("GITHUB_EVAL_ORG")
if not github_org:
print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
return False
# Repository configuration
owner = github_org
repo = "mcpmark-cicd"
headers = {
"Authorization": f"token {github_token}",
"Accept": "application/vnd.github.v3+json",
}
print("🔍 Starting Issue Management Workflow Verification")
print("=" * 60)
# Wait for workflows to complete
workflows_completed = _wait_for_workflow_completion(headers, owner, repo)
if not workflows_completed:
print(
"⚠️ Warning: Workflows may still be running. Continuing with verification..."
)
# Verify each test issue
all_passed = True
# 1. Verify bug issue
bug_ok, bug_errors = _verify_bug_issue(headers, owner, repo)
if not bug_ok:
all_passed = False
print("❌ Bug Issue Verification Failed:")
for error in bug_errors:
print(f" - {error}")
else:
print("✅ Bug Issue Verification Passed")
# 2. Verify epic issue
epic_ok, epic_errors = _verify_epic_issue(headers, owner, repo)
if not epic_ok:
all_passed = False
print("❌ Epic Issue Verification Failed:")
for error in epic_errors:
print(f" - {error}")
else:
print("✅ Epic Issue Verification Passed")
# 3. Verify maintenance issue
maintenance_ok, maintenance_errors = _verify_maintenance_issue(headers, owner, repo)
if not maintenance_ok:
all_passed = False
print("❌ Maintenance Issue Verification Failed:")
for error in maintenance_errors:
print(f" - {error}")
else:
print("✅ Maintenance Issue Verification Passed")
print("\n" + "=" * 60)
if all_passed:
print("🎉 All Issue Management Workflow verifications PASSED!")
print("\n📋 Summary:")
print(
" ✅ Bug issue: labels (including first-time-contributor), milestone, and auto-response verified"
)
print(
" ✅ Epic issue: labels, milestone, 4 sub-issues with checklist, and correct issue references verified"
)
print(
" ✅ Maintenance issue: labels, no milestone, and auto-response verified"
)
print("\n🤖 The GitHub Actions workflow automation is working correctly!")
else:
print("❌ Issue Management Workflow verification FAILED!")
print(" Some issues did not meet the expected automation requirements.")
return all_passed
if __name__ == "__main__":
success = verify()
sys.exit(0 if success else 1)