Deployment Status Workflow
L3
GithubMCPMark CI/CD
Create deployment status workflow with pre-deployment checks, rollback preparation, and comprehensive issue tracking for deployments.
Created by Zijian Wu
2025-08-15
Ci Cd AutomationWorkflow Automation
Model Ranking
Click on the dots to view the trajectory of each task run
Model | Run Results | Pass@4 | Pass^4 | Avg Time | Avg Turns | Input Tokens | Output Tokens | Total Tokens |
---|---|---|---|---|---|---|---|---|
claude-4-1-opus | 0 /1 | - | - | 184.5s | 1.0 | 16,640 | 153 | 16,793 |
claude-4-sonnet | 0 /4 | 216.1s | 3.0 | 63,333 | 3,069 | 66,402 | ||
deepseek-chat | 0 /4 | 286.3s | 6.3 | 126,896 | 4,952 | 131,848 | ||
gemini-2-5-pro | 0 /4 | 87.8s | 5.0 | 64,260 | 7,711 | 71,971 | ||
gpt-5 | 0 /4 | 309.1s | 7.5 | 147,061 | 23,282 | 170,343 | ||
grok-4 | 0 /4 | 30.1s | - | - | - | - | ||
k2 | 0 /4 | 135.5s | 2.5 | 45,829 | 1,466 | 47,295 | ||
o3 | 0 /4 | 117.2s | 5.0 | 65,832 | 5,071 | 70,904 | ||
qwen-3-coder | 0 /4 | 110.2s | 7.5 | 189,266 | 5,586 | 194,852 |
Task State
Instruction
Verify
Python
import sys
import os
import requests
import time
from typing import Dict, List, Optional, Tuple
from dotenv import load_dotenv
def _get_github_api(
endpoint: str, headers: Dict[str, str], owner: str, repo: str
) -> Tuple[bool, Optional[Dict]]:
"""Make a GET request to GitHub API and return (success, response)."""
url = f"https://api.github.com/repos/{owner}/{repo}/{endpoint}"
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
return True, response.json()
elif response.status_code == 404:
return False, None
else:
print(f"API error for {endpoint}: {response.status_code}", file=sys.stderr)
return False, None
except Exception as e:
print(f"Exception for {endpoint}: {e}", file=sys.stderr)
return False, None
def _search_github_issues(
query: str, headers: Dict[str, str]
) -> Tuple[bool, Optional[List]]:
"""Search GitHub issues using the search API."""
url = f"https://api.github.com/search/issues?q={query}&per_page=100"
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
data = response.json()
return True, data.get("items", [])
else:
print(f"Search API error: {response.status_code}", file=sys.stderr)
return False, None
except Exception as e:
print(f"Search exception: {e}", file=sys.stderr)
return False, None
def _wait_for_workflow_completion(
headers: Dict[str, str], owner: str, repo: str, max_wait: int = 300
) -> bool:
"""Wait for GitHub Actions workflows to complete processing."""
print("⏳ Waiting for deployment status workflows to complete...")
start_time = time.time()
no_workflow_check_count = 0
while time.time() - start_time < max_wait:
try:
# Check workflow runs for deployment-status.yml
success, response = _get_github_api(
"actions/workflows/deployment-status.yml/runs?per_page=10",
headers,
owner,
repo,
)
if success and response:
runs = response.get("workflow_runs", [])
if len(runs) > 0:
# Check status of recent runs
running_count = 0
completed_count = 0
failed_count = 0
for run in runs[:3]: # Check recent runs
status = run["status"]
conclusion = run.get("conclusion")
if status == "completed":
completed_count += 1
if conclusion == "failure":
failed_count += 1
elif status in ["in_progress", "queued"]:
running_count += 1
print(
f" Status: {completed_count} completed, {running_count} running/queued"
)
# Wait until NO workflows are running
if running_count == 0:
if failed_count > 0:
print(
f"⚠️ Warning: {failed_count} workflow runs failed, but continuing verification..."
)
print(
f"✅ All workflows completed. Found {completed_count} completed runs."
)
# Additional wait to ensure all processing is done
print(
"⏳ Additional wait for deployment processing to complete..."
)
time.sleep(5)
return True
else:
# No workflow runs found
no_workflow_check_count += 1
if no_workflow_check_count == 1:
print(
" No workflow runs found yet, waiting 5 seconds and checking once more..."
)
time.sleep(5)
continue
elif no_workflow_check_count >= 2:
print(
"⚠️ No workflow runs detected after 2 checks. Workflow may not have been triggered."
)
print(" Continuing with verification...")
return False
print(f"⏳ Still waiting... ({int(time.time() - start_time)}s elapsed)")
time.sleep(5)
except Exception as e:
print(f"⚠️ Error checking workflow status: {e}")
time.sleep(5)
print(f"⚠️ Workflow completion wait timed out after {max_wait}s")
return False
def _verify_workflow_runs(
headers: Dict[str, str], owner: str, repo: str
) -> Tuple[bool, List[str], Optional[Dict]]:
"""Verify that the deployment status workflow runs have the correct jobs."""
print("\n⚙️ Verifying deployment status workflow runs...")
errors = []
# Get the most recent workflow run
success, response = _get_github_api(
"actions/workflows/deployment-status.yml/runs?per_page=5",
headers,
owner,
repo,
)
if not success or not response:
return False, ["Failed to fetch workflow runs"], None
runs = response.get("workflow_runs", [])
if not runs:
return False, ["No workflow runs found for deployment-status.yml"], None
# Find the most recent successful run
latest_successful_run = None
for run in runs:
if run.get("conclusion") == "success":
latest_successful_run = run
break
if not latest_successful_run:
return False, ["No successful workflow runs found"], None
run_id = latest_successful_run["id"]
print(f" Found successful workflow run #{run_id}")
# Get jobs for this run
success, jobs_response = _get_github_api(
f"actions/runs/{run_id}/jobs", headers, owner, repo
)
if not success:
return False, ["Failed to fetch workflow jobs"], None
jobs = jobs_response.get("jobs", [])
expected_jobs = ["pre-deployment", "rollback-preparation", "post-deployment"]
found_jobs = [job["name"] for job in jobs]
missing_jobs = [job for job in expected_jobs if job not in found_jobs]
if missing_jobs:
errors.append(f"Missing jobs: {missing_jobs}. Found: {found_jobs}")
else:
print(f" ✅ All 3 required jobs found: {found_jobs}")
# Verify all jobs succeeded
failed_jobs = [job["name"] for job in jobs if job["conclusion"] != "success"]
if failed_jobs:
errors.append(f"Failed jobs: {failed_jobs}")
else:
print(" ✅ All jobs completed successfully")
# Verify sequential execution (each job should start after the previous one)
if len(jobs) >= 3:
job_times = {}
for job in jobs:
if job["name"] in expected_jobs and job["started_at"]:
job_times[job["name"]] = job["started_at"]
if len(job_times) >= 3:
# Check that jobs ran in correct sequence
import datetime
times = {
name: datetime.datetime.fromisoformat(time.replace("Z", "+00:00"))
for name, time in job_times.items()
}
# pre-deployment should start first
# rollback-preparation should start after pre-deployment
# post-deployment should start after rollback-preparation
if all(job in times for job in expected_jobs):
if (
times["rollback-preparation"] <= times["pre-deployment"]
or times["post-deployment"] <= times["rollback-preparation"]
):
errors.append("Jobs did not run in correct sequential order")
else:
print(" ✅ Jobs ran in correct sequential order")
else:
errors.append(
"Not enough job timing data to verify sequential execution"
)
return len(errors) == 0, errors, latest_successful_run
def _verify_deployment_issue(
run_data: Dict, headers: Dict[str, str], owner: str, repo: str
) -> Tuple[bool, List[str]]:
"""Verify that a deployment tracking issue was created and closed properly."""
print("\n📋 Verifying deployment tracking issue...")
errors = []
# Extract commit SHA from the workflow run
head_sha = run_data.get("head_sha")
if not head_sha:
return False, ["Could not determine head SHA from workflow run"]
short_sha = head_sha[:7]
expected_title = f"Deployment Tracking - {short_sha}"
# Search for the deployment tracking issue
success, issues = _search_github_issues(
f'repo:{owner}/{repo} "{expected_title}" is:issue', headers
)
if not success:
return False, ["Failed to search for deployment tracking issue"]
# Find the exact issue
deployment_issue = None
for issue in issues:
if issue.get("title") == expected_title:
deployment_issue = issue
break
if not deployment_issue:
return False, [f"Deployment tracking issue '{expected_title}' not found"]
issue_number = deployment_issue["number"]
print(f" Found deployment tracking issue #{issue_number}: {expected_title}")
# Check that issue is closed
if deployment_issue.get("state") != "closed":
errors.append(
f"Deployment issue #{issue_number} is not closed (state: {deployment_issue.get('state')})"
)
else:
print(f" ✅ Deployment issue #{issue_number} is closed")
# Check required labels
expected_labels = ["deployment", "completed"]
actual_labels = [label["name"] for label in deployment_issue.get("labels", [])]
missing_labels = [label for label in expected_labels if label not in actual_labels]
if missing_labels:
errors.append(
f"Missing labels on deployment issue: {missing_labels}. Found: {actual_labels}"
)
else:
print(f" ✅ Required labels found: {expected_labels}")
# Get issue comments to verify GitHub Actions bot comments
success, comments = _get_github_api(
f"issues/{issue_number}/comments", headers, owner, repo
)
if not success:
errors.append("Failed to get deployment issue comments")
return len(errors) == 0, errors
# Filter for GitHub Actions bot comments only
bot_comments = [
comment
for comment in comments
if comment.get("user", {}).get("login") == "github-actions[bot]"
]
if not bot_comments:
errors.append("No comments found from GitHub Actions bot")
return len(errors) == 0, errors
print(f" Found {len(bot_comments)} comment(s) from GitHub Actions bot")
# Get all bot comment bodies
bot_comment_bodies = [comment.get("body", "") for comment in bot_comments]
all_bot_comments = " ".join(bot_comment_bodies)
# Check for required GitHub Actions bot comment indicators
required_comment_indicators = [
"Pre-deployment checks completed",
"🔄 Rollback Plan Ready",
"Deployment Completed Successfully",
]
for indicator in required_comment_indicators:
if indicator not in all_bot_comments:
errors.append(
f"Missing required GitHub Actions bot comment indicator: '{indicator}'"
)
else:
print(f" ✅ Found GitHub Actions bot comment indicator: '{indicator}'")
# Find and verify the rollback plan comment from GitHub Actions bot
rollback_comment = None
for comment in bot_comments:
if "🔄 Rollback Plan Ready" in comment.get("body", ""):
rollback_comment = comment.get("body", "")
break
if rollback_comment:
print(" ✅ Found rollback plan comment from GitHub Actions bot")
# Check for required rollback plan elements
required_elements = [
"**Previous Commit**:",
"**Current Commit**:",
"**Package Version**:",
"✅ Executable rollback script created",
"✅ Configuration backups saved",
"✅ Dependency verification script prepared",
"✅ Comprehensive rollback documentation generated",
"✅ Compressed rollback package created",
"**SHA256**:",
"**Artifact**:",
"Quick Rollback Commands",
]
for element in required_elements:
if element not in rollback_comment:
errors.append(f"Missing element in rollback plan: '{element}'")
else:
print(f" ✅ Found rollback plan element: '{element}'")
# Verify commit SHAs in rollback comment
if f"**Current Commit**: {head_sha}" in rollback_comment:
print(f" ✅ Current commit SHA verified: {head_sha}")
else:
errors.append(
f"Current commit SHA {head_sha} not found in rollback comment"
)
# Extract and verify previous commit SHA
if "**Previous Commit**:" in rollback_comment:
import re
prev_sha_match = re.search(
r"\*\*Previous Commit\*\*:\s*([a-f0-9]{40})", rollback_comment
)
if prev_sha_match:
prev_sha = prev_sha_match.group(1)
print(f" ✅ Previous commit SHA found: {prev_sha}")
# Verify it's a valid 40-character SHA
if len(prev_sha) != 40:
errors.append(
f"Previous commit SHA has invalid length: {len(prev_sha)}"
)
else:
errors.append(
"Previous commit SHA format not found in rollback comment"
)
else:
errors.append("Previous commit SHA not found in rollback comment")
# Verify SHA256 checksum is present
sha256_match = re.search(r"\*\*SHA256\*\*:\s*([a-f0-9]{64})", rollback_comment)
if sha256_match:
sha256_value = sha256_match.group(1)
print(f" ✅ SHA256 checksum found: {sha256_value[:16]}...")
else:
errors.append(
"SHA256 checksum not found or invalid format in rollback comment"
)
else:
errors.append("Rollback plan comment not found from GitHub Actions bot")
return len(errors) == 0, errors
def verify() -> bool:
"""
Verify that the deployment status workflow automation is working correctly.
"""
# Load environment variables
load_dotenv(".mcp_env")
github_token = os.environ.get("MCP_GITHUB_TOKEN")
if not github_token:
print("Error: MCP_GITHUB_TOKEN environment variable not set", file=sys.stderr)
return False
# Get GitHub organization
github_org = os.environ.get("GITHUB_EVAL_ORG")
if not github_org:
print("Error: GITHUB_EVAL_ORG environment variable not set", file=sys.stderr)
return False
# Repository configuration
owner = github_org
repo = "mcpmark-cicd"
headers = {
"Authorization": f"token {github_token}",
"Accept": "application/vnd.github.v3+json",
}
print("🔍 Starting Deployment Status Workflow Verification")
print("=" * 60)
# Wait for workflows to complete
workflows_completed = _wait_for_workflow_completion(headers, owner, repo)
if not workflows_completed:
print(
"⚠️ Warning: Workflows may still be running. Continuing with verification..."
)
# Verify workflow runs and jobs
all_passed = True
# 1. Verify workflow runs have correct jobs
runs_ok, runs_errors, run_data = _verify_workflow_runs(headers, owner, repo)
if not runs_ok:
all_passed = False
print("❌ Workflow Runs Verification Failed:")
for error in runs_errors:
print(f" - {error}")
else:
print("✅ Workflow Runs Verification Passed")
# 2. Verify deployment issue if workflow runs passed
if run_data:
issue_ok, issue_errors = _verify_deployment_issue(
run_data, headers, owner, repo
)
if not issue_ok:
all_passed = False
print("❌ Deployment Issue Verification Failed:")
for error in issue_errors:
print(f" - {error}")
else:
print("✅ Deployment Issue Verification Passed")
print("\n" + "=" * 60)
if all_passed:
print("🎉 All Deployment Status Workflow verifications PASSED!")
print("\n📋 Summary:")
print(
" ✅ Workflow runs with correct 3 sequential jobs: pre-deployment, rollback-preparation, post-deployment"
)
print(" ✅ Deployment tracking issue created and closed with proper labels")
print(" ✅ Issue contains rollback plan with all required elements")
print(" ✅ Previous and current commit SHAs are correctly tracked")
print(" ✅ All workflow automation comments are present")
print(
"\n🤖 The GitHub Actions deployment status workflow is working correctly!"
)
else:
print("❌ Deployment Status Workflow verification FAILED!")
print(" Some components did not meet the expected automation requirements.")
return all_passed
if __name__ == "__main__":
success = verify()
sys.exit(0 if success else 1)