Linting Ci Workflow
Set up ESLint workflow for code quality enforcement on all pull requests with proper CI integration.
Model Ranking
Model | Run Results | Pass@4 | Pass^4 | Avg Time | Avg Turns | Input Tokens | Output Tokens | Total Tokens |
---|---|---|---|---|---|---|---|---|
Model | Run Results | Pass@4 | Pass^4 | Avg Time | Avg Turns | Input Tokens | Output Tokens | Total Tokens |
gpt-5-low | 4 /4 | 133.2s | 5.5 | 66,204 | 6,621 | 72,825 | ||
gpt-5-medium | 4 /4 | 130.8s | 5.5 | 60,213 | 7,906 | 68,119 | ||
o3 | 4 /4 | 59.0s | 5.3 | 62,682 | 2,163 | 64,845 | ||
o4-mini | 3 /4 | 67.0s | 5.5 | 70,500 | 3,850 | 74,349 | ||
gpt-5-high | 2 /4 | 447.3s | 7.8 | 94,080 | 21,895 | 115,975 | ||
gpt-5-nano-medium | 2 /4 | 60.9s | 5.0 | 57,283 | 8,614 | 65,897 | ||
grok-4 | 2 /4 | 112.8s | 5.8 | 106,833 | 1,093 | 110,467 | ||
claude-sonnet-4 | 1 /4 | 107.2s | 7.3 | 161,952 | 2,827 | 164,778 | ||
gpt-5-mini-high | 1 /4 | 116.7s | 10.0 | 151,173 | 9,205 | 160,378 | ||
gpt-5-nano-high | 1 /4 | 136.9s | 6.5 | 77,418 | 25,078 | 102,496 | ||
claude-opus-4-1 | 0 /1 | - | - | 174.8s | 7.0 | 155,317 | 2,781 | 158,098 |
claude-sonnet-4-high | 0 /4 | 64.5s | 7.0 | 137,966 | 2,467 | 140,433 | ||
claude-sonnet-4-low | 0 /4 | 85.4s | 8.8 | 184,549 | 2,974 | 187,522 | ||
deepseek-chat | 0 /4 | 162.4s | 9.0 | 178,381 | 1,957 | 180,338 | ||
gemini-2-5-flash | 0 /4 | 945.1s | 63.5 | 5,581,746 | 10,872 | 5,592,618 | ||
gemini-2-5-pro | 0 /4 | 229.4s | 11.0 | 232,406 | 7,232 | 239,637 | ||
glm-4-5 | 0 /4 | 77.5s | 8.8 | 188,551 | 2,535 | 191,085 | ||
gpt-4-1 | 0 /4 | 37.4s | 6.0 | 72,525 | 1,138 | 73,663 | ||
gpt-4-1-mini | 0 /4 | 60.2s | 10.5 | 142,520 | 1,415 | 143,935 | ||
gpt-4-1-nano | 0 /4 | 111.5s | 11.3 | 218,089 | 10,214 | 228,303 | ||
gpt-5-mini-low | 0 /4 | 50.5s | 10.5 | 171,591 | 1,852 | 173,443 | ||
gpt-5-mini-medium | 0 /4 | 49.3s | 9.5 | 134,865 | 2,667 | 137,532 | ||
gpt-5-nano-low | 0 /4 | 48.9s | 10.0 | 129,465 | 2,925 | 132,390 | ||
gpt-oss-120b | 0 /4 | 27.8s | 4.3 | 46,598 | 1,864 | 48,462 | ||
grok-code-fast-1 | 0 /4 | 47.2s | 8.3 | 143,102 | 4,650 | 147,752 | ||
kimi-k2-0711 | 0 /4 | 121.1s | 7.0 | 121,014 | 1,824 | 122,838 | ||
kimi-k2-0905 | 0 /4 | 194.5s | 7.0 | 105,745 | 1,611 | 107,355 | ||
qwen-3-coder-plus | 0 /4 | 84.4s | 17.5 | 520,693 | 2,418 | 523,111 | ||
qwen-3-max | 0 /4 | 75.5s | 12.5 | 251,040 | 1,518 | 252,557 |
Task State
Instruction
I need you to set up a proper linting workflow for our CI pipeline to ensure code quality standards are enforced on all pull requests. Here's what you need to do:
Step 1: Create Linting Configuration Branch Create a new branch called 'ci/add-eslint-workflow' from the main branch.
Step 2: Create ESLint Configuration
On the new branch, create the file .eslintrc.json
in the repository root with:
{
"env": {
"browser": true,
"es2021": true,
"node": true
},
"extends": [
"eslint:recommended"
],
"parserOptions": {
"ecmaVersion": 12,
"sourceType": "module"
},
"rules": {
"no-unused-vars": "error",
"no-console": "warn",
"semi": ["error", "always"],
"quotes": ["error", "single"]
},
"ignorePatterns": ["node_modules/", "dist/", "build/"]
}
Step 3: Create GitHub Actions Linting Workflow
Create the file .github/workflows/lint.yml
with:
- Workflow name: "Code Linting"
- Triggers on: push to main, pull_request events
- Uses ubuntu-latest runner
- Sets up Node.js version 18 using actions/setup-node
- Installs dependencies with npm ci
- Installs ESLint globally
- Runs ESLint on all JavaScript files in src/ directories
- Fails the workflow if linting errors are found
Step 4: Create a File That Will Fail Linting
Create the file src/example.js
with intentional linting violations that will cause the CI check to fail.
Step 5: Create Pull Request Commit all the changes (ESLint config, workflow file, and example file with linting errors) in a single commit, then create a pull request from 'ci/add-eslint-workflow' to 'main' with:
- Title: "Add ESLint workflow for code quality enforcement"
- Body must include:
- A "## Summary" heading describing the linting setup
- A "## Changes" heading listing the files added
- A "## Testing" heading explaining how to test the workflow
- Mention that the PR intentionally includes linting errors to demonstrate the workflow
Step 6: Fix Linting Errors and Update PR
Fix the linting errors in src/example.js
and commit the changes in a single commit to update the PR so that the CI check passes.
Verify
import sys
import os
import requests
from typing import Dict, List, Optional, Tuple
import base64
from dotenv import load_dotenv
import time
import json
def _get_github_api(
endpoint: str, headers: Dict[str, str], org: str, repo: str = "claude-code"
) -> Tuple[bool, Optional[Dict]]:
"""Make a GET request to GitHub API and return (success, response)."""
url = f"https://api.github.com/repos/{org}/{repo}/{endpoint}"
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
return True, response.json()
elif response.status_code == 404:
return False, None
else:
print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
return False, None
except Exception as e:
print(f"Exception for {endpoint}: {e}", file=sys.stderr)
return False, None
def _check_branch_exists(
branch_name: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> bool:
"""Verify that a branch exists in the repository."""
success, _ = _get_github_api(f"branches/{branch_name}", headers, org, repo)
return success
def _get_file_content(
file_path: str,
headers: Dict[str, str],
org: str,
repo: str = "claude-code",
ref: str = "main",
) -> Optional[str]:
"""Get the content of a file from the repository."""
success, result = _get_github_api(
f"contents/{file_path}?ref={ref}", headers, org, repo
)
if not success or not result:
return None
try:
content = base64.b64decode(result.get("content", "")).decode("utf-8")
return content
except Exception as e:
print(f"Content decode error for {file_path}: {e}", file=sys.stderr)
return None
def _find_pr_by_title_keyword(
keyword: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> Optional[Dict]:
"""Find a PR by title keyword and return the PR data."""
for state in ["open", "closed"]:
success, prs = _get_github_api(
f"pulls?state={state}&per_page=100", headers, org, repo
)
if success and prs:
for pr in prs:
if keyword.lower() in pr.get("title", "").lower():
return pr
return None
def _get_workflow_runs_for_pr(
pr_number: int, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
"""Get workflow runs for a specific PR."""
success, runs = _get_github_api(
"actions/runs?event=pull_request&per_page=100", headers, org, repo
)
if not success or not runs:
return []
pr_runs = []
for run in runs.get("workflow_runs", []):
# Check if this run is associated with our PR
for pr in run.get("pull_requests", []):
if pr.get("number") == pr_number:
pr_runs.append(run)
break
return pr_runs
def _get_pr_commits(
pr_number: int, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
"""Get commits for a specific PR."""
success, commits = _get_github_api(f"pulls/{pr_number}/commits", headers, org, repo)
if not success or not commits:
return []
return commits
def _get_workflow_runs_for_commit(
commit_sha: str, headers: Dict[str, str], org: str, repo: str = "mcpmark-cicd"
) -> List[Dict]:
"""Get workflow runs for a specific commit."""
success, runs = _get_github_api(
f"actions/runs?head_sha={commit_sha}&per_page=100", headers, org, repo
)
if not success or not runs:
return []
return runs.get("workflow_runs", [])
def verify() -> bool:
"""
Programmatically verify that the ESLint CI workflow setup
meets the requirements described in description.md.
"""
# Configuration constants
BRANCH_NAME = "ci/add-eslint-workflow"
PR_KEYWORD = "eslint workflow"
# Expected files and their content checks
ESLINT_CONFIG_PATH = ".eslintrc.json"
WORKFLOW_PATH = ".github/workflows/lint.yml"
EXAMPLE_FILE_PATH = "src/example.js"
# Expected workflow content keywords
WORKFLOW_KEYWORDS = [
"Code Linting",
"ubuntu-latest",
"actions/setup-node",
"npm ci",
"eslint",
"src/",
]
# Load environment variables from .mcp_env
load_dotenv(".mcp_env")
# Get GitHub token and org
github_token = os.environ.get("MCP_GITHUB_TOKEN")
github_org = os.environ.get("GITHUB_EVAL_ORG")
if not github_token:
print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
return False
if not github_org:
print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
return False
headers = {
"Authorization": f"Bearer {github_token}",
"Accept": "application/vnd.github.v3+json",
}
# Run verification checks
print("Verifying ESLint CI workflow setup...")
# 1. Check that branch exists
print("1. Verifying CI branch exists...")
if not _check_branch_exists(BRANCH_NAME, headers, github_org):
print(f"Error: Branch '{BRANCH_NAME}' not found", file=sys.stderr)
return False
print("✓ CI branch created")
# 2. Check ESLint configuration file
print("2. Verifying .eslintrc.json...")
eslint_content = _get_file_content(
ESLINT_CONFIG_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
)
if not eslint_content:
print("Error: .eslintrc.json not found", file=sys.stderr)
return False
# Validate ESLint config is valid JSON and contains required rules
try:
eslint_config = json.loads(eslint_content)
rules = eslint_config.get("rules", {})
required_rules = ["no-unused-vars", "semi", "quotes"]
missing_rules = [rule for rule in required_rules if rule not in rules]
if missing_rules:
print(
f"Error: .eslintrc.json missing rules: {missing_rules}", file=sys.stderr
)
return False
except json.JSONDecodeError:
print("Error: .eslintrc.json is not valid JSON", file=sys.stderr)
return False
print("✓ ESLint configuration created with proper rules")
# 3. Check GitHub Actions workflow file
print("3. Verifying .github/workflows/lint.yml...")
workflow_content = _get_file_content(
WORKFLOW_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
)
if not workflow_content:
print("Error: .github/workflows/lint.yml not found", file=sys.stderr)
return False
# Check workflow contains required keywords
missing_keywords = [kw for kw in WORKFLOW_KEYWORDS if kw not in workflow_content]
if missing_keywords:
print(f"Error: Workflow missing keywords: {missing_keywords}", file=sys.stderr)
return False
# Check trigger configuration
if "pull_request" not in workflow_content or "push" not in workflow_content:
print("Error: Workflow missing proper triggers", file=sys.stderr)
return False
print("✓ GitHub Actions workflow created with proper configuration")
# 4. Check example file with linting errors initially exists
print("4. Verifying src/example.js...")
example_content = _get_file_content(
EXAMPLE_FILE_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
)
if not example_content:
print("Error: src/example.js not found", file=sys.stderr)
return False
print("✓ Example file created")
# 5. Find and verify the linting PR
print("5. Verifying linting pull request...")
lint_pr = _find_pr_by_title_keyword(PR_KEYWORD, headers, github_org)
if not lint_pr:
# Try alternative keywords
lint_pr = _find_pr_by_title_keyword("eslint", headers, github_org)
if not lint_pr:
print("Error: Linting PR not found", file=sys.stderr)
return False
pr_body = lint_pr.get("body", "")
pr_number = lint_pr.get("number")
# Check PR body sections
required_sections = ["## Summary", "## Changes", "## Testing"]
missing_sections = [
section for section in required_sections if section not in pr_body
]
if missing_sections:
print(
f"Error: Linting PR missing sections: {missing_sections}", file=sys.stderr
)
return False
print("✓ Linting PR created with proper structure")
# 6. Check workflow runs and status changes
print("6. Verifying workflow execution and status...")
# First get the commits for this PR
commits = _get_pr_commits(pr_number, headers, github_org)
if len(commits) != 2:
print(
f"Error: Expected exactly 2 commits, found {len(commits)}", file=sys.stderr
)
return False
print("✓ Found exactly 2 commits as expected")
# Sort commits chronologically (oldest first)
commits.sort(key=lambda x: x.get("commit", {}).get("author", {}).get("date", ""))
first_commit_sha = commits[0].get("sha")
second_commit_sha = commits[1].get("sha")
print(f"First commit (should fail): {first_commit_sha[:7]}")
print(f"Second commit (should pass): {second_commit_sha[:7]}")
# Wait for workflows on both commits to complete
print("Waiting for workflow completion on first commit...")
first_commit_runs = []
second_commit_runs = []
start_time = time.time()
timeout = 120
no_workflow_check_count = 0
while time.time() - start_time < timeout:
first_commit_runs = _get_workflow_runs_for_commit(
first_commit_sha, headers, github_org
)
second_commit_runs = _get_workflow_runs_for_commit(
second_commit_sha, headers, github_org
)
# Check if any workflows exist
if not first_commit_runs and not second_commit_runs:
no_workflow_check_count += 1
if no_workflow_check_count == 1:
print(
"No workflow runs found yet, waiting 5 seconds and checking once more..."
)
time.sleep(5)
continue
elif no_workflow_check_count >= 2:
print(
"⚠️ No workflow runs detected after 2 checks. Workflows may not have been triggered."
)
print(" Continuing with verification...")
break
# Check if workflows are completed
first_completed = any(
run.get("status") == "completed" for run in first_commit_runs
)
second_completed = any(
run.get("status") == "completed" for run in second_commit_runs
)
if first_completed and second_completed:
break
print("Waiting for workflows to complete...")
time.sleep(10)
# Verify first commit workflow failed
first_commit_status = None
for run in first_commit_runs:
if run.get("status") == "completed":
conclusion = run.get("conclusion")
if conclusion in ["failure", "cancelled"]:
first_commit_status = "failed"
print("✓ First commit workflow failed as expected")
break
elif conclusion == "success":
first_commit_status = "passed"
break
if first_commit_status != "failed":
print(
"Error: First commit workflow should have failed due to linting errors",
file=sys.stderr,
)
return False
# Verify second commit workflow succeeded
second_commit_status = None
for run in second_commit_runs:
if run.get("status") == "completed":
conclusion = run.get("conclusion")
if conclusion == "success":
second_commit_status = "passed"
print("✓ Second commit workflow passed as expected")
break
elif conclusion in ["failure", "cancelled"]:
second_commit_status = "failed"
break
if second_commit_status != "passed":
print(
"Error: Second commit workflow should have passed after fixing linting errors",
file=sys.stderr,
)
return False
print(
"✓ Workflow status sequence verified: first commit failed → second commit passed"
)
# 7. Verify the final state shows clean code
print("7. Verifying final file state...")
final_example_content = _get_file_content(
EXAMPLE_FILE_PATH, headers, github_org, "mcpmark-cicd", BRANCH_NAME
)
if final_example_content:
# Check that obvious linting errors are fixed
if (
"unusedVariable" in final_example_content
or 'console.log("Hello World")' in final_example_content
):
print(
"Warning: Example file may still contain linting errors",
file=sys.stderr,
)
else:
print("✓ Linting errors appear to be fixed")
print("\n✅ All verification checks passed!")
print("ESLint CI workflow setup completed successfully:")
print(f" - Linting PR #{pr_number}")
print(f" - Branch: {BRANCH_NAME}")
print(
" - Files created: .eslintrc.json, .github/workflows/lint.yml, src/example.js"
)
print(" - Workflow configured for pull_request and push triggers")
print(
f" - Total workflow runs found: {len(first_commit_runs) + len(second_commit_runs)}"
)
print(
f" - First commit runs: {len(first_commit_runs)}, Second commit runs: {len(second_commit_runs)}"
)
return True
if __name__ == "__main__":
success = verify()
sys.exit(0 if success else 1)