Linting Ci Workflow
L3
GithubMCPMark CI/CD
Set up ESLint workflow for code quality enforcement on all pull requests with proper CI integration.
Created by Zijian Wu
2025-08-15
Ci Cd AutomationPr Workflows
Model Ranking
Model | Run Results | Pass@4 | Pass^4 | Avg Time | Avg Turns | Input Tokens | Output Tokens | Total Tokens |
---|---|---|---|---|---|---|---|---|
gpt-5 | 4 /4 | 133.2s | 5.5 | 66,204 | 6,621 | 72,825 | ||
o3 | 4 /4 | 59.0s | 5.3 | 62,682 | 2,163 | 64,845 | ||
claude-4-sonnet | 1 /4 | 107.2s | 7.3 | 161,952 | 2,827 | 164,778 | ||
claude-4-1-opus | 0 /1 | - | - | 174.8s | 7.0 | 155,317 | 2,781 | 158,098 |
deepseek-chat | 0 /4 | 162.4s | 9.0 | 178,381 | 1,957 | 180,338 | ||
gemini-2-5-pro | 0 /4 | 52.1s | 3.8 | 42,640 | 3,395 | 46,035 | ||
grok-4 | 0 /4 | 30.4s | - | - | - | - | ||
k2 | 0 /4 | 121.1s | 7.0 | 121,014 | 1,824 | 122,838 | ||
qwen-3-coder | 0 /4 | 68.5s | 9.8 | 242,032 | 1,973 | 244,005 |
Task State
Instruction
Verify
Python
import sys
import os
import requests
from typing import Dict, List, Optional, Tuple
import base64
from dotenv import load_dotenv
import time
import json
def _get_github_api(
endpoint: str, headers: Dict[str, str], org: str, repo: str = "claude-code"
) -> Tuple[bool, Optional[Dict]]:
"""Make a GET request to GitHub API and return (success, response)."""
url = f"https://api.github.com/repos/{org}/{repo}/{endpoint}"
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
return True, response.json()
elif response.status_code == 404:
return False, None
else:
print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
return False, None
except Exception as e:
print(f"Exception for {endpoint}: {e}", file=sys.stderr)
return False, None
def _check_branch_exists(
branch_name: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> bool:
"""Verify that a branch exists in the repository."""
success, _ = _get_github_api(f"branches/{branch_name}", headers, org, repo)
return success
def _get_file_content(
file_path: str,
headers: Dict[str, str],
org: str,
repo: str = "claude-code",
ref: str = "main",
) -> Optional[str]:
"""Get the content of a file from the repository."""
success, result = _get_github_api(
f"contents/{file_path}?ref={ref}", headers, org, repo
)
if not success or not result:
return None
try:
content = base64.b64decode(result.get("content", "")).decode("utf-8")
return content
except Exception as e:
print(f"Content decode error for {file_path}: {e}", file=sys.stderr)
return None
def _find_pr_by_title_keyword(
keyword: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> Optional[Dict]:
"""Find a PR by title keyword and return the PR data."""
for state in ["open", "closed"]:
success, prs = _get_github_api(
f"pulls?state={state}&per_page=100", headers, org, repo
)
if success and prs:
for pr in prs:
if keyword.lower() in pr.get("title", "").lower():
return pr
return None
def _get_workflow_runs_for_pr(
pr_number: int, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
"""Get workflow runs for a specific PR."""
success, runs = _get_github_api(
"actions/runs?event=pull_request&per_page=100", headers, org, repo
)
if not success or not runs:
return []
pr_runs = []
for run in runs.get("workflow_runs", []):
# Check if this run is associated with our PR
for pr in run.get("pull_requests", []):
if pr.get("number") == pr_number:
pr_runs.append(run)
break
return pr_runs
def _get_pr_commits(
pr_number: int, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
"""Get commits for a specific PR."""
success, commits = _get_github_api(f"pulls/{pr_number}/commits", headers, org, repo)
if not success or not commits:
return []
return commits
def _get_workflow_runs_for_commit(
commit_sha: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
"""Get workflow runs for a specific commit."""
success, runs = _get_github_api(
f"actions/runs?head_sha={commit_sha}&per_page=100", headers, org, repo
)
if not success or not runs:
return []
return runs.get("workflow_runs", [])
def verify() -> bool:
"""
Programmatically verify that the ESLint CI workflow setup
meets the requirements described in description.md.
"""
# Configuration constants
BRANCH_NAME = "ci/add-eslint-workflow"
PR_KEYWORD = "eslint workflow"
# Expected files and their content checks
ESLINT_CONFIG_PATH = ".eslintrc.json"
WORKFLOW_PATH = ".github/workflows/lint.yml"
EXAMPLE_FILE_PATH = "src/example.js"
# Expected workflow content keywords
WORKFLOW_KEYWORDS = [
"Code Linting",
"ubuntu-latest",
"actions/setup-node",
"npm ci",
"eslint",
"src/",
]
# Load environment variables from .mcp_env
load_dotenv(".mcp_env")
# Get GitHub token and org
github_token = os.environ.get("MCP_GITHUB_TOKEN")
github_org = os.environ.get("GITHUB_EVAL_ORG")
if not github_token:
print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
return False
if not github_org:
print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
return False
headers = {
"Authorization": f"Bearer {github_token}",
"Accept": "application/vnd.github.v3+json",
}
# Run verification checks
print("Verifying ESLint CI workflow setup...")
# 1. Check that branch exists
print("1. Verifying CI branch exists...")
if not _check_branch_exists(BRANCH_NAME, headers, github_org):
print(f"Error: Branch '{BRANCH_NAME}' not found", file=sys.stderr)
return False
print("✓ CI branch created")
# 2. Check ESLint configuration file
print("2. Verifying .eslintrc.json...")
eslint_content = _get_file_content(
ESLINT_CONFIG_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
)
if not eslint_content:
print("Error: .eslintrc.json not found", file=sys.stderr)
return False
# Validate ESLint config is valid JSON and contains required rules
try:
eslint_config = json.loads(eslint_content)
rules = eslint_config.get("rules", {})
required_rules = ["no-unused-vars", "semi", "quotes"]
missing_rules = [rule for rule in required_rules if rule not in rules]
if missing_rules:
print(
f"Error: .eslintrc.json missing rules: {missing_rules}", file=sys.stderr
)
return False
except json.JSONDecodeError:
print("Error: .eslintrc.json is not valid JSON", file=sys.stderr)
return False
print("✓ ESLint configuration created with proper rules")
# 3. Check GitHub Actions workflow file
print("3. Verifying .github/workflows/lint.yml...")
workflow_content = _get_file_content(
WORKFLOW_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
)
if not workflow_content:
print("Error: .github/workflows/lint.yml not found", file=sys.stderr)
return False
# Check workflow contains required keywords
missing_keywords = [kw for kw in WORKFLOW_KEYWORDS if kw not in workflow_content]
if missing_keywords:
print(f"Error: Workflow missing keywords: {missing_keywords}", file=sys.stderr)
return False
# Check trigger configuration
if "pull_request" not in workflow_content or "push" not in workflow_content:
print("Error: Workflow missing proper triggers", file=sys.stderr)
return False
print("✓ GitHub Actions workflow created with proper configuration")
# 4. Check example file with linting errors initially exists
print("4. Verifying src/example.js...")
example_content = _get_file_content(
EXAMPLE_FILE_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
)
if not example_content:
print("Error: src/example.js not found", file=sys.stderr)
return False
print("✓ Example file created")
# 5. Find and verify the linting PR
print("5. Verifying linting pull request...")
lint_pr = _find_pr_by_title_keyword(PR_KEYWORD, headers, github_org)
if not lint_pr:
# Try alternative keywords
lint_pr = _find_pr_by_title_keyword("eslint", headers, github_org)
if not lint_pr:
print("Error: Linting PR not found", file=sys.stderr)
return False
pr_body = lint_pr.get("body", "")
pr_number = lint_pr.get("number")
# Check PR body sections
required_sections = ["## Summary", "## Changes", "## Testing"]
missing_sections = [
section for section in required_sections if section not in pr_body
]
if missing_sections:
print(
f"Error: Linting PR missing sections: {missing_sections}", file=sys.stderr
)
return False
print("✓ Linting PR created with proper structure")
# 6. Check workflow runs and status changes
print("6. Verifying workflow execution and status...")
# First get the commits for this PR
commits = _get_pr_commits(pr_number, headers, github_org)
if len(commits) != 2:
print(
f"Error: Expected exactly 2 commits, found {len(commits)}", file=sys.stderr
)
return False
print("✓ Found exactly 2 commits as expected")
# Sort commits chronologically (oldest first)
commits.sort(key=lambda x: x.get("commit", {}).get("author", {}).get("date", ""))
first_commit_sha = commits[0].get("sha")
second_commit_sha = commits[1].get("sha")
print(f"First commit (should fail): {first_commit_sha[:7]}")
print(f"Second commit (should pass): {second_commit_sha[:7]}")
# Wait for workflows on both commits to complete
print("Waiting for workflow completion on first commit...")
first_commit_runs = []
second_commit_runs = []
start_time = time.time()
timeout = 120
no_workflow_check_count = 0
while time.time() - start_time < timeout:
first_commit_runs = _get_workflow_runs_for_commit(
first_commit_sha, headers, github_org
)
second_commit_runs = _get_workflow_runs_for_commit(
second_commit_sha, headers, github_org
)
# Check if any workflows exist
if not first_commit_runs and not second_commit_runs:
no_workflow_check_count += 1
if no_workflow_check_count == 1:
print(
"No workflow runs found yet, waiting 5 seconds and checking once more..."
)
time.sleep(5)
continue
elif no_workflow_check_count >= 2:
print(
"⚠️ No workflow runs detected after 2 checks. Workflows may not have been triggered."
)
print(" Continuing with verification...")
break
# Check if workflows are completed
first_completed = any(
run.get("status") == "completed" for run in first_commit_runs
)
second_completed = any(
run.get("status") == "completed" for run in second_commit_runs
)
if first_completed and second_completed:
break
print("Waiting for workflows to complete...")
time.sleep(10)
# Verify first commit workflow failed
first_commit_status = None
for run in first_commit_runs:
if run.get("status") == "completed":
conclusion = run.get("conclusion")
if conclusion in ["failure", "cancelled"]:
first_commit_status = "failed"
print("✓ First commit workflow failed as expected")
break
elif conclusion == "success":
first_commit_status = "passed"
break
if first_commit_status != "failed":
print(
"Error: First commit workflow should have failed due to linting errors",
file=sys.stderr,
)
return False
# Verify second commit workflow succeeded
second_commit_status = None
for run in second_commit_runs:
if run.get("status") == "completed":
conclusion = run.get("conclusion")
if conclusion == "success":
second_commit_status = "passed"
print("✓ Second commit workflow passed as expected")
break
elif conclusion in ["failure", "cancelled"]:
second_commit_status = "failed"
break
if second_commit_status != "passed":
print(
"Error: Second commit workflow should have passed after fixing linting errors",
file=sys.stderr,
)
return False
print(
"✓ Workflow status sequence verified: first commit failed → second commit passed"
)
# 7. Verify the final state shows clean code
print("7. Verifying final file state...")
final_example_content = _get_file_content(
EXAMPLE_FILE_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
)
if final_example_content:
# Check that obvious linting errors are fixed
if (
"unusedVariable" in final_example_content
or 'console.log("Hello World")' in final_example_content
):
print(
"Warning: Example file may still contain linting errors",
file=sys.stderr,
)
else:
print("✓ Linting errors appear to be fixed")
print("\n✅ All verification checks passed!")
print("ESLint CI workflow setup completed successfully:")
print(f" - Linting PR #{pr_number}")
print(f" - Branch: {BRANCH_NAME}")
print(
" - Files created: .eslintrc.json, .github/workflows/lint.yml, src/example.js"
)
print(" - Workflow configured for pull_request and push triggers")
print(
f" - Total workflow runs found: {len(first_commit_runs) + len(second_commit_runs)}"
)
print(
f" - First commit runs: {len(first_commit_runs)}, Second commit runs: {len(second_commit_runs)}"
)
return True
if __name__ == "__main__":
success = verify()
sys.exit(0 if success else 1)