Fix Conflict

L3
ModelContextProtocolGithubHarmony

Resolve merge conflicts by creating missing infrastructure and ensuring conflicted PRs can merge cleanly.

Created by Zijian Wu
2025-08-15
Pr Workflows

Model Ranking

Click on the dots to view the trajectory of each task run
Model
Run Results
Pass@4
Pass^4
Avg Time
Avg Turns
Input Tokens
Output Tokens
Total Tokens
OpenAI
gpt-5
3
/4
296.8s
17.5
1,143,982
12,409
1,156,391
Claude
claude-4-1-opus
0
/1
--
382.3s
14.0
869,307
4,839
874,146
Claude
claude-4-sonnet
0
/4
393.9s
24.8
1,975,114
8,267
1,983,382
DeepSeek
deepseek-chat
0
/4
371.9s
19.8
1,351,906
3,460
1,355,365
Gemini
gemini-2-5-pro
0
/4
96.3s
3.3
22,421
8,047
30,468
Grok
grok-4
0
/4
30.3s
-
-
-
-
MoonshotAI
k2
0
/4
306.1s
21.3
1,255,569
2,810
1,258,379
OpenAI
o3
0
/4
142.7s
16.8
821,691
4,236
825,927
Qwen
qwen-3-coder
0
/4
218.3s
22.8
1,531,806
2,976
1,534,782

Task State


Instruction



Verify

*.py
Python
import sys
import os
import requests
from typing import Dict, Optional, Tuple
from dotenv import load_dotenv


def _get_github_api(
    endpoint: str, headers: Dict[str, str], org: str, repo: str = "harmony"
) -> Tuple[bool, Optional[Dict]]:
    """Make a GET request to GitHub API and return (success, response)."""
    url = f"https://api.github.com/repos/{org}/{repo}/{endpoint}"
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return True, response.json()
        elif response.status_code == 404:
            return False, None
        else:
            print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
            return False, None
    except Exception as e:
        print(f"Exception for {endpoint}: {e}", file=sys.stderr)
        return False, None


def _check_ci_file_exists(
    file_path: str, headers: Dict[str, str], org: str, repo: str = "harmony"
) -> bool:
    """Check if CI file exists in main branch."""
    success, _ = _get_github_api(f"contents/{file_path}?ref=main", headers, org, repo)
    return success


def _check_pr_comments(
    pr_number: int,
    infra_pr_number: int,
    headers: Dict[str, str],
    org: str,
    repo: str = "harmony",
) -> bool:
    """Check if PR has a comment linking to the infrastructure PR using 'PR #[NUMBER]' format."""
    success, comments = _get_github_api(
        f"issues/{pr_number}/comments", headers, org, repo
    )
    if not success or not comments:
        return False

    # Look for "PR #123" pattern (case insensitive)
    import re

    for comment in comments:
        body = comment.get("body", "")
        if re.search(rf"PR\s*#{infra_pr_number}", body, re.IGNORECASE):
            return True
    return False


def _find_infrastructure_pr(
    headers: Dict[str, str], org: str, repo: str = "harmony"
) -> Optional[Dict]:
    """Find the infrastructure PR by checking title and body content."""
    success, prs = _get_github_api("pulls?state=all&per_page=50", headers, org, repo)
    if success and prs:
        for pr in prs:
            title = pr.get("title", "").lower()
            body = pr.get("body", "").lower()

            # Check title contains required keywords
            title_ok = "add ci infrastructure" in title and "resolve conflicts" in title

            # Check body contains required elements
            has_reference = "fixes #" in body or "resolves #" in body
            has_prep_text = "prepares infrastructure" in body
            has_github_text = "missing .github directory" in body
            has_workflow_text = "workflow conflicts" in body

            body_ok = (
                has_reference
                and has_prep_text
                and has_github_text
                and has_workflow_text
            )

            if title_ok and body_ok:
                return pr
    return None


def verify() -> bool:
    """
    Programmatically verify that the merge conflict resolution workflow meets the
    requirements described in description.md.
    """
    # Load environment variables from .mcp_env
    load_dotenv(".mcp_env")

    # Get GitHub token and org
    github_token = os.environ.get("MCP_GITHUB_TOKEN")
    github_org = os.environ.get("GITHUB_EVAL_ORG")

    if not github_token:
        print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
        return False

    if not github_org:
        print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
        return False

    headers = {
        "Authorization": f"token {github_token}",
        "Accept": "application/vnd.github.v3+json",
    }

    # Run verification checks
    print("Verifying merge conflict resolution workflow completion...")

    # 1. Check that CI infrastructure file exists in main (extracted from conflicted PR)
    print("1. Checking CI infrastructure was added to main...")
    # Check for both CI.yml and ci.yml (case-insensitive)
    ci_exists = _check_ci_file_exists(".github/workflows/CI.yml", headers, github_org)
    if not ci_exists:
        ci_exists = _check_ci_file_exists(".github/workflows/ci.yml", headers, github_org)
    
    if not ci_exists:
        print("Error: Neither .github/workflows/CI.yml nor .github/workflows/ci.yml found in main", file=sys.stderr)
        return False

    # 2. Find infrastructure PR with required title and body content
    print("2. Finding infrastructure PR with required content...")
    infra_pr = _find_infrastructure_pr(headers, github_org)
    if not infra_pr:
        print(
            "Error: No infrastructure PR found with required title and body content",
            file=sys.stderr,
        )
        print(
            "Required title: 'Add CI infrastructure' and 'resolve conflicts'",
            file=sys.stderr,
        )
        print(
            "Required body: reference with 'Fixes #' or 'Resolves #', 'prepares infrastructure', 'missing .github directory', 'workflow conflicts'",
            file=sys.stderr,
        )
        return False

    print(f"Found infrastructure PR #{infra_pr.get('number')}: {infra_pr.get('title')}")

    # 3. Check that infrastructure PR is merged
    if not infra_pr.get("merged_at"):
        print(
            f"Error: Infrastructure PR #{infra_pr.get('number')} not merged yet",
            file=sys.stderr,
        )
        return False

    # 4. Check that PR #24 is merged
    print("3. Checking that PR #24 is merged...")
    success, pr24 = _get_github_api("pulls/24", headers, github_org)
    if not success or not pr24:
        print("Error: PR #24 not found", file=sys.stderr)
        return False

    if not pr24.get("merged_at"):
        print("Error: PR #24 is not merged yet", file=sys.stderr)
        return False

    # 5. Check that PR #24 has a comment linking to the infrastructure PR
    print("4. Checking that PR #24 has comment linking to infrastructure PR...")
    if not _check_pr_comments(24, infra_pr.get("number"), headers, github_org):
        print(
            f"Error: PR #24 missing comment linking to infrastructure PR #{infra_pr.get('number')}",
            file=sys.stderr,
        )
        return False

    print("\n✓ Task completed successfully!")
    print(
        f"Infrastructure PR #{infra_pr.get('number')} extracted content from PR #24 and resolved conflicts"
    )
    print(
        "PR #24 is now merged cleanly and has a comment linking to the infrastructure PR"
    )
    return True


if __name__ == "__main__":
    success = verify()
    sys.exit(0 if success else 1)