Release Management Workflow

L3
ModelContextProtocolGithubHarmony

Implement comprehensive release management workflow including bug fixes, version updates, changelog creation, and PR merging.

Created by Zijian Wu
2025-08-15
Release CoordinationPr Workflows

Model Ranking

Click on the dots to view the trajectory of each task run
Model
Run Results
Pass@4
Pass^4
Avg Time
Avg Turns
Input Tokens
Output Tokens
Total Tokens
Claude
claude-4-1-opus
0
/1
--
340.4s
9.0
544,580
1,227
545,807
Claude
claude-4-sonnet
0
/4
143.9s
7.5
427,345
1,010
428,354
DeepSeek
deepseek-chat
0
/4
293.2s
8.8
487,481
468
487,949
Gemini
gemini-2-5-pro
0
/4
30.6s
1.0
6,482
1,545
8,027
OpenAI
gpt-5
0
/4
543.4s
14.3
872,763
31,047
903,809
Grok
grok-4
0
/4
29.9s
-
-
-
-
MoonshotAI
k2
0
/4
160.3s
3.8
134,371
305
134,676
OpenAI
o3
0
/4
150.8s
12.5
376,488
6,604
383,092
Qwen
qwen-3-coder
0
/4
288.7s
9.3
466,850
10,080
476,930

Task State


Instruction



Verify

*.py
Python
import sys
import os
import requests
from typing import Dict, List, Optional, Tuple
import base64
from dotenv import load_dotenv


def _get_github_api(
    endpoint: str, headers: Dict[str, str], org: str, repo: str = "harmony"
) -> Tuple[bool, Optional[Dict]]:
    """Make a GET request to GitHub API and return (success, response)."""
    url = f"https://api.github.com/repos/{org}/{repo}/{endpoint}"
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return True, response.json()
        elif response.status_code == 404:
            return False, None
        else:
            print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
            return False, None
    except Exception as e:
        print(f"Exception for {endpoint}: {e}", file=sys.stderr)
        return False, None


def _check_branch_exists(
    branch_name: str, headers: Dict[str, str], org: str, repo: str = "harmony"
) -> bool:
    """Verify that a branch exists in the repository."""
    success, _ = _get_github_api(f"branches/{branch_name}", headers, org, repo)
    return success


def _check_file_content(
    branch: str,
    file_path: str,
    keywords: List[str],
    headers: Dict[str, str],
    org: str,
    repo: str = "harmony",
) -> bool:
    """Verify that a file exists in branch and contains required keywords."""
    success, result = _get_github_api(
        f"contents/{file_path}?ref={branch}", headers, org, repo
    )
    if not success or not result:
        return False

    if keywords and result.get("content"):
        try:
            content = base64.b64decode(result.get("content", "")).decode("utf-8")
            return all(keyword in content for keyword in keywords)
        except Exception as e:
            print(f"Content decode error for {file_path}: {e}", file=sys.stderr)
            return False

    return True


def _check_specific_file_content(
    branch: str,
    file_path: str,
    expected_content: str,
    headers: Dict[str, str],
    org: str,
    repo: str = "harmony",
    min_length: int = 100,
) -> bool:
    """Verify that a file contains specific exact content and has reasonable size."""
    success, result = _get_github_api(
        f"contents/{file_path}?ref={branch}", headers, org, repo
    )
    if not success or not result:
        return False

    if result.get("content"):
        try:
            content = base64.b64decode(result.get("content", "")).decode("utf-8")
            # Check both that expected content exists and file has reasonable content
            return expected_content in content and len(content) >= min_length
        except Exception as e:
            print(f"Content decode error for {file_path}: {e}", file=sys.stderr)
            return False

    return False


def _check_pr_merged(
    title_substring: str,
    base_branch: str,
    headers: Dict[str, str],
    org: str,
    repo: str = "harmony",
) -> Tuple[bool, Optional[int]]:
    """Check if a PR with specified title was merged into base branch and return PR number."""
    # Check closed PRs to find merged ones
    success, prs = _get_github_api(
        "pulls?state=closed&per_page=100", headers, org, repo
    )
    if not success or not prs:
        return False, None

    for pr in prs:
        title_match = title_substring.lower() in pr.get("title", "").lower()
        base_match = pr.get("base", {}).get("ref") == base_branch
        is_merged = pr.get("merged_at") is not None

        if title_match and base_match and is_merged:
            return True, pr.get("number")

    return False, None


def _check_pr_squash_merged(
    pr_number: int, headers: Dict[str, str], org: str, repo: str = "harmony"
) -> bool:
    """Check if a PR was merged using squash and merge method."""
    # Get the PR details
    success, pr = _get_github_api(f"pulls/{pr_number}", headers, org, repo)
    if not success or not pr:
        return False

    if not pr.get("merged_at"):
        return False

    merge_commit_sha = pr.get("merge_commit_sha")
    if not merge_commit_sha:
        return False

    # Get the merge commit details
    success, commit = _get_github_api(f"commits/{merge_commit_sha}", headers, org, repo)
    if not success or not commit:
        return False

    # For squash and merge, the commit will have exactly one parent
    # and the commit message typically includes the PR number
    parents = commit.get("parents", [])
    commit_message = commit.get("commit", {}).get("message", "")

    # Squash and merge commits have exactly 1 parent (the base branch)
    # Regular merge commits have 2 parents (base and head branches)
    if len(parents) == 1 and f"#{pr_number}" in commit_message:
        return True

    return False


def verify() -> bool:
    """
    Programmatically verify that the release management workflow meets the
    requirements described in description.md.
    """
    # Load environment variables from .mcp_env
    load_dotenv(".mcp_env")

    # Get GitHub token and org
    github_token = os.environ.get("MCP_GITHUB_TOKEN")
    github_org = os.environ.get("GITHUB_EVAL_ORG")

    if not github_token:
        print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
        return False

    if not github_org:
        print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
        return False

    # Configuration constants
    RELEASE_BRANCH = "release-v1.1.0"

    # Expected content checks with minimum file sizes to ensure files aren't just stubs
    METASEP_FIX = 'FormattingToken::MetaSep => "<|meta_sep|>"'
    REGISTRY_FIX = '(FormattingToken::MetaSep, "<|meta_sep|>")'
    METAEND_FIX = '(FormattingToken::MetaEnd, "<|meta_end|>")'
    UTILS_CONTENT = "export function cn(...inputs: ClassValue[])"
    GITIGNORE_ADDITION = "!demo/harmony-demo/src/lib"
    VERSION_110 = 'version = "1.1.0"'

    CHANGELOG_KEYWORDS = [
        "## [1.1.0] - 2025-08-07",
        "MetaSep token mapping bug",
        "shadcn utils.ts file",
        "Fixed MetaSep token",
        "Registry now properly recognizes",
    ]

    headers = {
        "Authorization": f"token {github_token}",
        "Accept": "application/vnd.github.v3+json",
    }

    # Run verification checks
    print("Verifying GitHub release management workflow completion...")

    # 1. Check release branch exists
    print("1. Verifying release branch exists...")
    if not _check_branch_exists(RELEASE_BRANCH, headers, github_org):
        print(f"Error: Branch '{RELEASE_BRANCH}' not found", file=sys.stderr)
        return False

    # 2. Check MetaSep fix in encoding.rs (with min content length to ensure file wasn't gutted)
    print("2. Verifying MetaSep token fix in encoding.rs...")
    if not _check_specific_file_content(
        "main", "src/encoding.rs", METASEP_FIX, headers, github_org, min_length=500
    ):
        print(
            "Error: MetaSep token fix not found in src/encoding.rs or file is too small",
            file=sys.stderr,
        )
        return False

    # 3. Check registry updates (both MetaSep and MetaEnd)
    print("3. Verifying MetaSep and MetaEnd registry additions...")
    if not _check_specific_file_content(
        "main", "src/registry.rs", REGISTRY_FIX, headers, github_org, min_length=500
    ):
        print(
            "Error: MetaSep registry fix not found in src/registry.rs or file is too small",
            file=sys.stderr,
        )
        return False
    if not _check_specific_file_content(
        "main", "src/registry.rs", METAEND_FIX, headers, github_org, min_length=500
    ):
        print(
            "Error: MetaEnd registry fix not found in src/registry.rs", file=sys.stderr
        )
        return False

    # 4. Check utils.ts file exists with correct content
    print("4. Verifying shadcn utils.ts file...")
    if not _check_specific_file_content(
        "main",
        "demo/harmony-demo/src/lib/utils.ts",
        UTILS_CONTENT,
        headers,
        github_org,
        min_length=50,
    ):
        print("Error: utils.ts file not found or incorrect content", file=sys.stderr)
        return False

    # 5. Check .gitignore update
    print("5. Verifying .gitignore update...")
    if not _check_specific_file_content(
        "main", ".gitignore", GITIGNORE_ADDITION, headers, github_org, min_length=100
    ):
        print("Error: .gitignore update not found", file=sys.stderr)
        return False

    # 6. Check version update in Cargo.toml only (pyproject.toml uses dynamic versioning)
    print("6. Verifying version update in Cargo.toml...")
    if not _check_specific_file_content(
        "main", "Cargo.toml", VERSION_110, headers, github_org, min_length=200
    ):
        print("Error: Version 1.1.0 not found in Cargo.toml", file=sys.stderr)
        return False

    # 7. Check CHANGELOG.md exists with required content
    print("7. Verifying CHANGELOG.md...")
    if not _check_file_content(
        "main", "CHANGELOG.md", CHANGELOG_KEYWORDS, headers, github_org
    ):
        print(
            "Error: CHANGELOG.md not found or missing required content", file=sys.stderr
        )
        return False

    # 8. Check release PR was merged and get PR number
    print("8. Verifying release pull request was merged...")
    pr_merged, pr_number = _check_pr_merged(
        "Release v1.1.0", "main", headers, github_org
    )
    if not pr_merged:
        print("Error: Release pull request not found or not merged", file=sys.stderr)
        return False

    # 9. Check PR was merged using squash and merge
    print("9. Verifying pull request was merged using 'squash and merge' method...")
    if pr_number and not _check_pr_squash_merged(pr_number, headers, github_org):
        print(
            f"Error: Pull request #{pr_number} was not merged using 'squash and merge' method",
            file=sys.stderr,
        )
        return False

    print("\n✓ All verification checks passed!")
    print("Release management workflow completed successfully.")
    return True


if __name__ == "__main__":
    success = verify()
    sys.exit(0 if success else 1)