Assign Contributor Labels

L3
ModelContextProtocolGithubMissing Semester

Assign labels to open issues and PRs based on contributors mentioned in comments or the most frequent contributor from past 100 commits, using assigned-username format.

Created by Zijian Wu
2025-08-15
Issue ManagementLabel AutomationContributor Analysis

Model Ranking

Click on the dots to view the trajectory of each task run
Model
Run Results
Pass@4
Pass^4
Avg Time
Avg Turns
Input Tokens
Output Tokens
Total Tokens
Claude
claude-4-1-opus
0
/1
--
110.9s
3.0
370,537
829
371,366
Claude
claude-4-sonnet
0
/4
57.9s
3.0
367,281
527
367,808
DeepSeek
deepseek-chat
0
/4
18.2s
1.0
14,284
114
14,398
Gemini
gemini-2-5-pro
0
/4
44.0s
1.0
5,101
2,957
8,058
OpenAI
gpt-5
0
/4
352.0s
6.8
910,545
13,915
924,459
Grok
grok-4
0
/4
30.6s
-
-
-
-
MoonshotAI
k2
0
/4
125.9s
3.0
273,629
154
273,783
OpenAI
o3
0
/4
105.7s
7.5
961,841
1,558
963,398
Qwen
qwen-3-coder
0
/4
345.7s
16.8
2,878,691
2,410
2,881,101

Task State


Instruction



Verify

*.py
Python
import sys
import os
import requests
from typing import Dict, Optional, Tuple, List
from dotenv import load_dotenv


def _get_github_api(
    endpoint: str, headers: Dict[str, str], org: str, repo: str = "missing-semester"
) -> Tuple[bool, Optional[Dict]]:
    """Make a GET request to GitHub API and return (success, response)."""
    url = f"https://api.github.com/repos/{org}/{repo}/{endpoint}"
    
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return True, response.json()
        elif response.status_code == 404:
            return False, None
        else:
            print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
            return False, None
    except Exception as e:
        print(f"Exception for {endpoint}: {e}", file=sys.stderr)
        return False, None


def _get_issue_labels(
    issue_number: int,
    headers: Dict[str, str],
    org: str,
    repo: str = "missing-semester"
) -> Optional[List[str]]:
    """Get labels for a specific issue/PR."""
    success, result = _get_github_api(f"issues/{issue_number}", headers, org, repo)
    if not success or not result:
        return None
    
    labels = result.get("labels", [])
    return [label["name"] for label in labels]


def verify() -> bool:
    """
    Programmatically verify that the labels were assigned correctly to issues and PRs.
    """
    # Load environment variables from .mcp_env
    load_dotenv(".mcp_env")

    # Get GitHub token and org
    github_token = os.environ.get("MCP_GITHUB_TOKEN")
    github_org = os.environ.get("GITHUB_EVAL_ORG")

    if not github_token:
        print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
        return False

    if not github_org:
        print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
        return False

    headers = {
        "Authorization": f"Bearer {github_token}",
        "Accept": "application/vnd.github.v3+json",
    }

    print("Verifying contributor labels assignment task completion...")

    # Expected labels configuration
    expected_labels = {
        # Issues
        9: ["assigned-jonhoo", "assigned-anishathalye"],  # Issue #9
        14: ["assigned-jonhoo", "assigned-anishathalye"],  # Issue #14
        15: ["assigned-anishathalye"],  # Issue #15
        # PRs
        21: ["assigned-anishathalye"],  # PR #21
        22: ["assigned-anishathalye"],  # PR #22
        23: ["assigned-anishathalye"],  # PR #23
        24: ["assigned-anishathalye"],  # PR #24
    }

    all_passed = True

    for item_number, expected in expected_labels.items():
        item_type = "Issue" if item_number in [9, 14, 15] else "PR"
        print(f"\nChecking {item_type} #{item_number}...")
        
        labels = _get_issue_labels(item_number, headers, github_org, "missing-semester")
        
        if labels is None:
            print(f"  ❌ Failed to retrieve {item_type} #{item_number}", file=sys.stderr)
            all_passed = False
            continue
        
        # Sort both lists for comparison
        labels_sorted = sorted(labels)
        expected_sorted = sorted(expected)
        
        if labels_sorted == expected_sorted:
            print(f"  ✅ {item_type} #{item_number} has correct labels: {labels_sorted}")
        else:
            print(f"  ❌ {item_type} #{item_number} has incorrect labels", file=sys.stderr)
            print(f"     Expected: {expected_sorted}", file=sys.stderr)
            print(f"     Found: {labels_sorted}", file=sys.stderr)
            all_passed = False

    if all_passed:
        print("\n✅ All verification checks passed!")
        print("Contributor labels assignment task completed successfully:")
        print("  - Issues #9 and #14 have both 'assigned-jonhoo' and 'assigned-anishathalye' labels")
        print("  - Issue #15 and all 4 open PRs have 'assigned-anishathalye' label")
    else:
        print("\n❌ Some verification checks failed", file=sys.stderr)

    return all_passed


if __name__ == "__main__":
    success = verify()
    sys.exit(0 if success else 1)