Label Color Standardization

L3
ModelContextProtocolGithubClaude Code

Standardize label colors from default gray to a comprehensive color scheme for better visual organization and issue triage.

Created by Zijian Wu
2025-08-15
Issue ManagementWorkflow Automation

Model Ranking

Click on the dots to view the trajectory of each task run
Model
Run Results
Pass@4
Pass^4
Avg Time
Avg Turns
Input Tokens
Output Tokens
Total Tokens
Claude
claude-4-1-opus
0
/1
--
337.8s
13.0
651,976
3,885
655,861
Claude
claude-4-sonnet
0
/4
190.5s
12.3
1,062,279
4,634
1,066,913
DeepSeek
deepseek-chat
0
/4
221.3s
9.3
568,260
2,184
570,444
Gemini
gemini-2-5-pro
0
/4
59.2s
3.8
48,392
3,025
51,416
OpenAI
gpt-5
0
/4
310.1s
11.8
598,326
14,163
612,489
Grok
grok-4
0
/4
30.8s
-
-
-
-
MoonshotAI
k2
0
/4
223.8s
6.8
325,726
1,322
327,048
OpenAI
o3
0
/4
84.9s
9.0
220,935
3,295
224,229
Qwen
qwen-3-coder
0
/4
136.8s
11.5
934,621
1,485
936,105

Task State


Instruction



Verify

*.py
Python
import sys
import os
import requests
from typing import Dict, List, Optional, Tuple
from dotenv import load_dotenv


def _get_github_api(
    endpoint: str, headers: Dict[str, str], org: str, repo: str = "claude-code"
) -> Tuple[bool, Optional[Dict]]:
    """Make a GET request to GitHub API and return (success, response)."""
    url = f"https://api.github.com/repos/{org}/{repo}/{endpoint}"
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return True, response.json()
        elif response.status_code == 404:
            return False, None
        else:
            print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
            return False, None
    except Exception as e:
        print(f"Exception for {endpoint}: {e}", file=sys.stderr)
        return False, None



def _check_branch_exists(
    branch_name: str, headers: Dict[str, str], org: str, repo: str = "claude-code"
) -> bool:
    """Verify that a branch exists in the repository."""
    success, _ = _get_github_api(f"branches/{branch_name}", headers, org, repo)
    return success


def _check_file_content(
    branch: str,
    file_path: str,
    headers: Dict[str, str],
    org: str,
    repo: str = "claude-code",
) -> Optional[str]:
    """Get file content from a branch."""
    import base64

    success, result = _get_github_api(
        f"contents/{file_path}?ref={branch}", headers, org, repo
    )
    if not success or not result:
        return None

    if result.get("content"):
        try:
            content = base64.b64decode(result.get("content", "")).decode("utf-8")
            return content
        except Exception as e:
            print(f"Content decode error for {file_path}: {e}", file=sys.stderr)
            return None

    return None


def _parse_label_table(content: str) -> List[str]:
    """Parse the label table from markdown content and return label names."""
    documented_labels = []

    # Find the table in the content
    lines = content.split("\n")
    in_table = False

    for line in lines:
        # Skip header and separator lines
        if "| Label Name | Category |" in line:
            in_table = True
            continue
        if in_table and line.startswith("|---"):
            continue

        # Parse table rows
        if in_table and line.startswith("|"):
            parts = [p.strip() for p in line.split("|")]
            if len(parts) >= 3:  # Should have at least label, category
                label_name = parts[1].strip()
                if label_name:
                    documented_labels.append(label_name)

        # Stop at end of table
        if in_table and line and not line.startswith("|"):
            break

    return documented_labels


def _find_issue_by_title_keywords(
    title_keywords: List[str],
    headers: Dict[str, str],
    org: str,
    repo: str = "claude-code",
) -> Optional[Dict]:
    """Find an issue by title keywords and return the issue data."""
    for state in ["open", "closed"]:
        success, issues = _get_github_api(
            f"issues?state={state}&per_page=100", headers, org, repo
        )
        if success and issues:
            for issue in issues:
                # Skip pull requests
                if "pull_request" in issue:
                    continue
                title = issue.get("title", "").lower()
                if all(keyword.lower() in title for keyword in title_keywords):
                    return issue
    return None


def _find_pr_by_title_keywords(
    title_keywords: List[str],
    headers: Dict[str, str],
    org: str,
    repo: str = "claude-code",
) -> Optional[Dict]:
    """Find a PR by title keywords and return the PR data."""
    for state in ["open", "closed"]:
        success, prs = _get_github_api(
            f"pulls?state={state}&per_page=100", headers, org, repo
        )
        if success and prs:
            for pr in prs:
                title = pr.get("title", "").lower()
                if all(keyword.lower() in title for keyword in title_keywords):
                    return pr
    return None


def _get_issue_comments(
    issue_number: int, headers: Dict[str, str], org: str, repo: str = "claude-code"
) -> List[Dict]:
    """Get all comments for an issue."""
    success, comments = _get_github_api(
        f"issues/{issue_number}/comments", headers, org, repo
    )
    if success and comments:
        return comments
    return []




def verify() -> bool:
    """
    Programmatically verify that the label color standardization workflow meets the
    requirements described in description.md.
    """
    # Load environment variables from .mcp_env
    load_dotenv(".mcp_env")

    # Get GitHub token and org
    github_token = os.environ.get("MCP_GITHUB_TOKEN")
    github_org = os.environ.get("GITHUB_EVAL_ORG")

    if not github_token:
        print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
        return False

    if not github_org:
        print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
        return False

    # Configuration constants
    BRANCH_NAME = "feat/label-color-guide"

    # Issue requirements
    ISSUE_TITLE_KEYWORDS = ["Document label organization", "label guide"]
    ISSUE_KEYWORDS = [
        "label documentation",
        "visual organization",
        "label guide",
        "organization",
    ]

    # PR requirements
    PR_TITLE_KEYWORDS = ["label organization guide", "visual organization"]
    PR_KEYWORDS = [
        "label documentation",
        "organization guide",
        "visual improvement",
        "documentation",
    ]

    # All expected labels in the repository that are actually used/discoverable via MCP tools
    # Note: Excludes 'wontfix', 'invalid', 'good first issue', 'help wanted' as they exist
    # in the repository but are not used by any issues (not discoverable via MCP search)
    ALL_EXPECTED_LABELS = [
        "bug",
        "enhancement",
        "duplicate",
        "question",
        "documentation",
        "platform:macos",
        "platform:linux",
        "platform:windows",
        "area:core",
        "area:tools",
        "area:tui",
        "area:ide",
        "area:mcp",
        "area:api",
        "area:security",
        "area:model",
        "area:auth",
        "area:packaging",
        "has repro",
        "memory",
        "perf:memory",
        "external",
    ]

    headers = {
        "Authorization": f"token {github_token}",
        "Accept": "application/vnd.github.v3+json",
    }

    # Run verification checks
    print("Verifying label color standardization workflow completion...")

    # 1. Check that feature branch exists
    print("1. Verifying feature branch exists...")
    if not _check_branch_exists(BRANCH_NAME, headers, github_org):
        print(f"Error: Branch '{BRANCH_NAME}' not found", file=sys.stderr)
        return False

    # 2. Check documentation file exists and has correct format
    print("2. Verifying label documentation file...")
    doc_content = _check_file_content(
        BRANCH_NAME, "docs/LABEL_COLORS.md", headers, github_org
    )
    if not doc_content:
        print("Error: docs/LABEL_COLORS.md not found", file=sys.stderr)
        return False

    # Parse the label table from documentation
    documented_labels = _parse_label_table(doc_content)
    if len(documented_labels) < 20:
        print(
            f"Error: Documentation table incomplete, found only {len(documented_labels)} labels",
            file=sys.stderr,
        )
        return False

    # 3. Verify labels are documented
    print("3. Verifying expected labels are documented...")
    print(f"  ✓ {len(ALL_EXPECTED_LABELS)} expected labels defined for verification")

    # 4. Find the created issue
    print("4. Verifying issue creation...")
    issue = _find_issue_by_title_keywords(ISSUE_TITLE_KEYWORDS, headers, github_org)
    if not issue:
        print(
            "Error: Issue with title containing required keywords not found",
            file=sys.stderr,
        )
        return False

    issue_number = issue.get("number")
    issue_body = issue.get("body", "")

    # Check issue content has required sections and keywords
    issue_required_sections = ["## Problem", "## Proposed Solution", "## Benefits"]
    for section in issue_required_sections:
        if section not in issue_body:
            print(f"Error: Issue body missing required section: {section}", file=sys.stderr)
            return False

    # Check issue has required keywords
    if not all(keyword.lower() in issue_body.lower() for keyword in ISSUE_KEYWORDS):
        missing_keywords = [kw for kw in ISSUE_KEYWORDS if kw.lower() not in issue_body.lower()]
        print(f"Error: Issue body missing required keywords: {missing_keywords}", file=sys.stderr)
        return False

    # Check issue has initial required labels (enhancement and documentation)
    issue_label_names = [label["name"] for label in issue.get("labels", [])]
    initial_required_labels = ["enhancement", "documentation"]
    for required_label in initial_required_labels:
        if required_label not in issue_label_names:
            print(f"Error: Issue missing initial required label: {required_label}", file=sys.stderr)
            return False

    # 5. Find the created PR
    print("5. Verifying pull request creation...")
    pr = _find_pr_by_title_keywords(PR_TITLE_KEYWORDS, headers, github_org)
    if not pr:
        print(
            "Error: PR with title containing required keywords not found",
            file=sys.stderr,
        )
        return False

    pr_number = pr.get("number")
    pr_body = pr.get("body", "")
    pr_labels = pr.get("labels", [])

    # Check PR references issue with correct pattern
    if f"Fixes #{issue_number}" not in pr_body and f"fixes #{issue_number}" not in pr_body:
        print(f"Error: PR does not contain 'Fixes #{issue_number}' pattern", file=sys.stderr)
        return False

    # Check PR body has required sections and keywords
    pr_required_sections = ["## Summary", "## Changes", "## Verification"]
    for section in pr_required_sections:
        if section not in pr_body:
            print(f"Error: PR body missing required section: {section}", file=sys.stderr)
            return False

    # Check PR has required keywords
    if not all(keyword.lower() in pr_body.lower() for keyword in PR_KEYWORDS):
        missing_keywords = [kw for kw in PR_KEYWORDS if kw.lower() not in pr_body.lower()]
        print(f"Error: PR body missing required keywords: {missing_keywords}", file=sys.stderr)
        return False

    # Check PR has sufficient labels (at least 5 from different categories)
    if len(pr_labels) < 5:
        print(f"Error: PR has only {len(pr_labels)} labels, needs at least 5", file=sys.stderr)
        return False

    # 6. Verify issue has ALL expected/usable labels applied (demonstrates organization)
    print("6. Verifying issue has all expected labels applied...")
    issue_label_names = [label["name"] for label in issue.get("labels", [])]
    # Use our expected labels list instead of all repo labels (excludes unused labels)
    expected_labels_to_check = ALL_EXPECTED_LABELS
    missing_labels = []

    for expected_label in expected_labels_to_check:
        if expected_label not in issue_label_names:
            missing_labels.append(expected_label)

    if missing_labels:
        print(
            f"Error: Issue missing {len(missing_labels)} expected labels: {missing_labels[:5]}...",
            file=sys.stderr,
        )
        return False

    print(f"  ✓ Issue has all {len(expected_labels_to_check)} expected labels applied")

    # 7. Verify issue has comment documenting changes
    print("7. Verifying issue comment with documentation...")
    issue_comments = _get_issue_comments(issue_number, headers, github_org)

    found_update_comment = False
    comment_required_keywords = ["documentation created", "label guide complete", "organization complete"]
    
    for comment in issue_comments:
        body = comment.get("body", "")
        # Check for PR reference and required keywords
        if (f"PR #{pr_number}" in body and 
            any(keyword.lower() in body.lower() for keyword in comment_required_keywords) and
            "total" in body.lower() and "labels" in body.lower()):
            found_update_comment = True
            break

    if not found_update_comment:
        print("Error: Issue missing comment documenting changes with required content", file=sys.stderr)
        print("  Comment should include: PR reference, label count, and completion keywords", file=sys.stderr)
        return False

    # 8. Final verification of complete workflow
    print("8. Final verification of workflow completion...")
    
    # Skip repository label existence check - we trust that our expected labels 
    # are the ones actually discoverable/usable via MCP tools

    # Ensure expected labels are documented (not all repo labels, since some are unused)
    documented_label_count = len(documented_labels)
    expected_label_count = len(ALL_EXPECTED_LABELS)

    if documented_label_count < expected_label_count:
        print(
            f"Error: Documentation incomplete - {documented_label_count} documented vs {expected_label_count} expected",
            file=sys.stderr,
        )
        return False

    # Check that all expected labels are documented
    missing_documented_labels = []
    for expected_label in ALL_EXPECTED_LABELS:
        if expected_label not in documented_labels:
            missing_documented_labels.append(expected_label)

    if missing_documented_labels:
        print(
            f"Error: Documentation missing expected labels: {missing_documented_labels}",
            file=sys.stderr,
        )
        return False

    print(f"  ✓ All {expected_label_count} expected labels documented")
    print(f"  ✓ All {len(ALL_EXPECTED_LABELS)} expected labels present and documented")

    print("\n✓ All verification checks passed!")
    print("Label documentation workflow completed successfully:")
    print(
        f"  - Issue #{issue_number}: {issue.get('title')} (with all {len(issue_label_names)} labels)"
    )
    print(f"  - PR #{pr_number}: {pr.get('title')}")
    print(f"  - Branch: {BRANCH_NAME}")
    print("  - Documentation: docs/LABEL_COLORS.md")
    print(f"  - {expected_label_count} labels documented for better organization")
    return True


if __name__ == "__main__":
    success = verify()
    sys.exit(0 if success else 1)