Linting Ci Workflow

L3
ModelContextProtocolGithubMCPMark CI/CD

Set up ESLint workflow for code quality enforcement on all pull requests with proper CI integration.

Created by Zijian Wu
2025-08-15
Ci Cd AutomationPr Workflows

Model Ranking

Model
Run Results
Pass@4
Pass^4
Avg Time
Avg Turns
Input Tokens
Output Tokens
Total Tokens
OpenAI
gpt-5
4
/4
133.2s
5.5
66,204
6,621
72,825
OpenAI
o3
4
/4
59.0s
5.3
62,682
2,163
64,845
Claude
claude-4-sonnet
1
/4
107.2s
7.3
161,952
2,827
164,778
Claude
claude-4-1-opus
0
/1
--
174.8s
7.0
155,317
2,781
158,098
DeepSeek
deepseek-chat
0
/4
162.4s
9.0
178,381
1,957
180,338
Gemini
gemini-2-5-pro
0
/4
52.1s
3.8
42,640
3,395
46,035
Grok
grok-4
0
/4
30.4s
-
-
-
-
MoonshotAI
k2
0
/4
121.1s
7.0
121,014
1,824
122,838
Qwen
qwen-3-coder
0
/4
68.5s
9.8
242,032
1,973
244,005

Task State


Instruction



Verify

*.py
Python
import sys
import os
import requests
from typing import Dict, List, Optional, Tuple
import base64
from dotenv import load_dotenv
import time
import json


def _get_github_api(
    endpoint: str, headers: Dict[str, str], org: str, repo: str = "claude-code"
) -> Tuple[bool, Optional[Dict]]:
    """Make a GET request to GitHub API and return (success, response)."""
    url = f"https://api.github.com/repos/{org}/{repo}/{endpoint}"

    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return True, response.json()
        elif response.status_code == 404:
            return False, None
        else:
            print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
            return False, None
    except Exception as e:
        print(f"Exception for {endpoint}: {e}", file=sys.stderr)
        return False, None


def _check_branch_exists(
    branch_name: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> bool:
    """Verify that a branch exists in the repository."""
    success, _ = _get_github_api(f"branches/{branch_name}", headers, org, repo)
    return success


def _get_file_content(
    file_path: str,
    headers: Dict[str, str],
    org: str,
    repo: str = "claude-code",
    ref: str = "main",
) -> Optional[str]:
    """Get the content of a file from the repository."""
    success, result = _get_github_api(
        f"contents/{file_path}?ref={ref}", headers, org, repo
    )
    if not success or not result:
        return None

    try:
        content = base64.b64decode(result.get("content", "")).decode("utf-8")
        return content
    except Exception as e:
        print(f"Content decode error for {file_path}: {e}", file=sys.stderr)
        return None


def _find_pr_by_title_keyword(
    keyword: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> Optional[Dict]:
    """Find a PR by title keyword and return the PR data."""
    for state in ["open", "closed"]:
        success, prs = _get_github_api(
            f"pulls?state={state}&per_page=100", headers, org, repo
        )
        if success and prs:
            for pr in prs:
                if keyword.lower() in pr.get("title", "").lower():
                    return pr
    return None


def _get_workflow_runs_for_pr(
    pr_number: int, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
    """Get workflow runs for a specific PR."""
    success, runs = _get_github_api(
        "actions/runs?event=pull_request&per_page=100", headers, org, repo
    )
    if not success or not runs:
        return []

    pr_runs = []
    for run in runs.get("workflow_runs", []):
        # Check if this run is associated with our PR
        for pr in run.get("pull_requests", []):
            if pr.get("number") == pr_number:
                pr_runs.append(run)
                break

    return pr_runs


def _get_pr_commits(
    pr_number: int, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
    """Get commits for a specific PR."""
    success, commits = _get_github_api(f"pulls/{pr_number}/commits", headers, org, repo)
    if not success or not commits:
        return []
    return commits


def _get_workflow_runs_for_commit(
    commit_sha: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
    """Get workflow runs for a specific commit."""
    success, runs = _get_github_api(
        f"actions/runs?head_sha={commit_sha}&per_page=100", headers, org, repo
    )
    if not success or not runs:
        return []
    return runs.get("workflow_runs", [])


def verify() -> bool:
    """
    Programmatically verify that the ESLint CI workflow setup
    meets the requirements described in description.md.
    """
    # Configuration constants
    BRANCH_NAME = "ci/add-eslint-workflow"
    PR_KEYWORD = "eslint workflow"

    # Expected files and their content checks
    ESLINT_CONFIG_PATH = ".eslintrc.json"
    WORKFLOW_PATH = ".github/workflows/lint.yml"
    EXAMPLE_FILE_PATH = "src/example.js"

    # Expected workflow content keywords
    WORKFLOW_KEYWORDS = [
        "Code Linting",
        "ubuntu-latest",
        "actions/setup-node",
        "npm ci",
        "eslint",
        "src/",
    ]

    # Load environment variables from .mcp_env
    load_dotenv(".mcp_env")

    # Get GitHub token and org
    github_token = os.environ.get("MCP_GITHUB_TOKEN")
    github_org = os.environ.get("GITHUB_EVAL_ORG")

    if not github_token:
        print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
        return False

    if not github_org:
        print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
        return False

    headers = {
        "Authorization": f"Bearer {github_token}",
        "Accept": "application/vnd.github.v3+json",
    }

    # Run verification checks
    print("Verifying ESLint CI workflow setup...")

    # 1. Check that branch exists
    print("1. Verifying CI branch exists...")
    if not _check_branch_exists(BRANCH_NAME, headers, github_org):
        print(f"Error: Branch '{BRANCH_NAME}' not found", file=sys.stderr)
        return False
    print("✓ CI branch created")

    # 2. Check ESLint configuration file
    print("2. Verifying .eslintrc.json...")
    eslint_content = _get_file_content(
        ESLINT_CONFIG_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
    )
    if not eslint_content:
        print("Error: .eslintrc.json not found", file=sys.stderr)
        return False

    # Validate ESLint config is valid JSON and contains required rules
    try:
        eslint_config = json.loads(eslint_content)
        rules = eslint_config.get("rules", {})

        required_rules = ["no-unused-vars", "semi", "quotes"]
        missing_rules = [rule for rule in required_rules if rule not in rules]
        if missing_rules:
            print(
                f"Error: .eslintrc.json missing rules: {missing_rules}", file=sys.stderr
            )
            return False

    except json.JSONDecodeError:
        print("Error: .eslintrc.json is not valid JSON", file=sys.stderr)
        return False

    print("✓ ESLint configuration created with proper rules")

    # 3. Check GitHub Actions workflow file
    print("3. Verifying .github/workflows/lint.yml...")
    workflow_content = _get_file_content(
        WORKFLOW_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
    )
    if not workflow_content:
        print("Error: .github/workflows/lint.yml not found", file=sys.stderr)
        return False

    # Check workflow contains required keywords
    missing_keywords = [kw for kw in WORKFLOW_KEYWORDS if kw not in workflow_content]
    if missing_keywords:
        print(f"Error: Workflow missing keywords: {missing_keywords}", file=sys.stderr)
        return False

    # Check trigger configuration
    if "pull_request" not in workflow_content or "push" not in workflow_content:
        print("Error: Workflow missing proper triggers", file=sys.stderr)
        return False

    print("✓ GitHub Actions workflow created with proper configuration")

    # 4. Check example file with linting errors initially exists
    print("4. Verifying src/example.js...")
    example_content = _get_file_content(
        EXAMPLE_FILE_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
    )
    if not example_content:
        print("Error: src/example.js not found", file=sys.stderr)
        return False

    print("✓ Example file created")

    # 5. Find and verify the linting PR
    print("5. Verifying linting pull request...")
    lint_pr = _find_pr_by_title_keyword(PR_KEYWORD, headers, github_org)
    if not lint_pr:
        # Try alternative keywords
        lint_pr = _find_pr_by_title_keyword("eslint", headers, github_org)

    if not lint_pr:
        print("Error: Linting PR not found", file=sys.stderr)
        return False

    pr_body = lint_pr.get("body", "")
    pr_number = lint_pr.get("number")

    # Check PR body sections
    required_sections = ["## Summary", "## Changes", "## Testing"]
    missing_sections = [
        section for section in required_sections if section not in pr_body
    ]
    if missing_sections:
        print(
            f"Error: Linting PR missing sections: {missing_sections}", file=sys.stderr
        )
        return False

    print("✓ Linting PR created with proper structure")

    # 6. Check workflow runs and status changes
    print("6. Verifying workflow execution and status...")

    # First get the commits for this PR
    commits = _get_pr_commits(pr_number, headers, github_org)
    if len(commits) != 2:
        print(
            f"Error: Expected exactly 2 commits, found {len(commits)}", file=sys.stderr
        )
        return False

    print("✓ Found exactly 2 commits as expected")

    # Sort commits chronologically (oldest first)
    commits.sort(key=lambda x: x.get("commit", {}).get("author", {}).get("date", ""))

    first_commit_sha = commits[0].get("sha")
    second_commit_sha = commits[1].get("sha")

    print(f"First commit (should fail): {first_commit_sha[:7]}")
    print(f"Second commit (should pass): {second_commit_sha[:7]}")

    # Wait for workflows on both commits to complete
    print("Waiting for workflow completion on first commit...")
    first_commit_runs = []
    second_commit_runs = []

    start_time = time.time()
    timeout = 120
    no_workflow_check_count = 0

    while time.time() - start_time < timeout:
        first_commit_runs = _get_workflow_runs_for_commit(
            first_commit_sha, headers, github_org
        )
        second_commit_runs = _get_workflow_runs_for_commit(
            second_commit_sha, headers, github_org
        )

        # Check if any workflows exist
        if not first_commit_runs and not second_commit_runs:
            no_workflow_check_count += 1
            if no_workflow_check_count == 1:
                print(
                    "No workflow runs found yet, waiting 5 seconds and checking once more..."
                )
                time.sleep(5)
                continue
            elif no_workflow_check_count >= 2:
                print(
                    "⚠️ No workflow runs detected after 2 checks. Workflows may not have been triggered."
                )
                print("   Continuing with verification...")
                break

        # Check if workflows are completed
        first_completed = any(
            run.get("status") == "completed" for run in first_commit_runs
        )
        second_completed = any(
            run.get("status") == "completed" for run in second_commit_runs
        )

        if first_completed and second_completed:
            break

        print("Waiting for workflows to complete...")
        time.sleep(10)

    # Verify first commit workflow failed
    first_commit_status = None
    for run in first_commit_runs:
        if run.get("status") == "completed":
            conclusion = run.get("conclusion")
            if conclusion in ["failure", "cancelled"]:
                first_commit_status = "failed"
                print("✓ First commit workflow failed as expected")
                break
            elif conclusion == "success":
                first_commit_status = "passed"
                break

    if first_commit_status != "failed":
        print(
            "Error: First commit workflow should have failed due to linting errors",
            file=sys.stderr,
        )
        return False

    # Verify second commit workflow succeeded
    second_commit_status = None
    for run in second_commit_runs:
        if run.get("status") == "completed":
            conclusion = run.get("conclusion")
            if conclusion == "success":
                second_commit_status = "passed"
                print("✓ Second commit workflow passed as expected")
                break
            elif conclusion in ["failure", "cancelled"]:
                second_commit_status = "failed"
                break

    if second_commit_status != "passed":
        print(
            "Error: Second commit workflow should have passed after fixing linting errors",
            file=sys.stderr,
        )
        return False

    print(
        "✓ Workflow status sequence verified: first commit failed → second commit passed"
    )

    # 7. Verify the final state shows clean code
    print("7. Verifying final file state...")
    final_example_content = _get_file_content(
        EXAMPLE_FILE_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
    )

    if final_example_content:
        # Check that obvious linting errors are fixed
        if (
            "unusedVariable" in final_example_content
            or 'console.log("Hello World")' in final_example_content
        ):
            print(
                "Warning: Example file may still contain linting errors",
                file=sys.stderr,
            )
        else:
            print("✓ Linting errors appear to be fixed")

    print("\n✅ All verification checks passed!")
    print("ESLint CI workflow setup completed successfully:")
    print(f"  - Linting PR #{pr_number}")
    print(f"  - Branch: {BRANCH_NAME}")
    print(
        "  - Files created: .eslintrc.json, .github/workflows/lint.yml, src/example.js"
    )
    print("  - Workflow configured for pull_request and push triggers")
    print(
        f"  - Total workflow runs found: {len(first_commit_runs) + len(second_commit_runs)}"
    )
    print(
        f"  - First commit runs: {len(first_commit_runs)}, Second commit runs: {len(second_commit_runs)}"
    )

    return True


if __name__ == "__main__":
    success = verify()
    sys.exit(0 if success else 1)