301 lines
8.5 KiB
Python
301 lines
8.5 KiB
Python
"""
|
|
Jenkins task processing
|
|
Asynchronous Jenkins job triggering using Celery
|
|
"""
|
|
|
|
import asyncio
|
|
import time
|
|
from typing import Dict, Any
|
|
from datetime import datetime
|
|
import structlog
|
|
from celery import Celery, Task
|
|
import httpx
|
|
|
|
from app.config import get_settings
|
|
from app.services.jenkins_service import JenkinsService
|
|
|
|
logger = structlog.get_logger()
|
|
settings = get_settings()
|
|
|
|
# Create Celery app
|
|
celery_app = Celery(
|
|
"gitea_webhook_ambassador",
|
|
broker=settings.redis.url,
|
|
backend=settings.redis.url,
|
|
include=["app.tasks.jenkins_tasks"]
|
|
)
|
|
|
|
# Celery configuration
|
|
celery_app.conf.update(
|
|
task_serializer="json",
|
|
accept_content=["json"],
|
|
result_serializer="json",
|
|
timezone="UTC",
|
|
enable_utc=True,
|
|
task_track_started=True,
|
|
task_time_limit=300, # 5 minutes timeout
|
|
task_soft_time_limit=240, # 4 minutes soft timeout
|
|
worker_prefetch_multiplier=1,
|
|
worker_max_tasks_per_child=1000,
|
|
worker_max_memory_per_child=200000, # 200MB
|
|
task_acks_late=True,
|
|
task_reject_on_worker_lost=True,
|
|
task_always_eager=False, # Set to False in production
|
|
result_expires=3600, # Result cache 1 hour
|
|
)
|
|
|
|
|
|
class JenkinsTask(Task):
|
|
"""Jenkins task base class"""
|
|
|
|
abstract = True
|
|
|
|
def __init__(self):
|
|
self.jenkins_service = None
|
|
|
|
def __call__(self, *args, **kwargs):
|
|
if self.jenkins_service is None:
|
|
self.jenkins_service = JenkinsService()
|
|
return self.run(*args, **kwargs)
|
|
|
|
def on_failure(self, exc, task_id, args, kwargs, einfo):
|
|
"""Task failure callback"""
|
|
logger.error("Task failed",
|
|
task_id=task_id,
|
|
task_name=self.name,
|
|
error=str(exc),
|
|
args=args,
|
|
kwargs=kwargs)
|
|
|
|
def on_retry(self, exc, task_id, args, kwargs, einfo):
|
|
"""Task retry callback"""
|
|
logger.warning("Task retrying",
|
|
task_id=task_id,
|
|
task_name=self.name,
|
|
error=str(exc),
|
|
retry_count=self.request.retries)
|
|
|
|
def on_success(self, retval, task_id, args, kwargs):
|
|
"""Task success callback"""
|
|
logger.info("Task completed successfully",
|
|
task_id=task_id,
|
|
task_name=self.name,
|
|
result=retval)
|
|
|
|
|
|
@celery_app.task(
|
|
bind=True,
|
|
base=JenkinsTask,
|
|
max_retries=3,
|
|
default_retry_delay=60,
|
|
autoretry_for=(Exception,),
|
|
retry_backoff=True,
|
|
retry_jitter=True
|
|
)
|
|
def trigger_jenkins_job(
|
|
self,
|
|
job_name: str,
|
|
jenkins_url: str,
|
|
parameters: Dict[str, str],
|
|
event_id: str,
|
|
repository: str,
|
|
branch: str,
|
|
commit_hash: str,
|
|
priority: int = 1
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Trigger Jenkins job
|
|
Args:
|
|
job_name: Jenkins job name
|
|
jenkins_url: Jenkins URL
|
|
parameters: job parameters
|
|
event_id: event ID
|
|
repository: repository name
|
|
branch: branch name
|
|
commit_hash: commit hash
|
|
priority: priority
|
|
Returns:
|
|
Dict: job execution result
|
|
"""
|
|
start_time = time.time()
|
|
|
|
try:
|
|
logger.info("Starting Jenkins job trigger",
|
|
task_id=self.request.id,
|
|
job_name=job_name,
|
|
jenkins_url=jenkins_url,
|
|
repository=repository,
|
|
branch=branch,
|
|
commit_hash=commit_hash,
|
|
priority=priority)
|
|
|
|
# Create Jenkins service instance
|
|
jenkins_service = JenkinsService()
|
|
|
|
# Trigger Jenkins job
|
|
result = asyncio.run(jenkins_service.trigger_job(
|
|
job_name=job_name,
|
|
jenkins_url=jenkins_url,
|
|
parameters=parameters
|
|
))
|
|
|
|
execution_time = time.time() - start_time
|
|
|
|
if result["success"]:
|
|
logger.info("Jenkins job triggered successfully",
|
|
task_id=self.request.id,
|
|
job_name=job_name,
|
|
build_number=result.get("build_number"),
|
|
execution_time=execution_time)
|
|
|
|
return {
|
|
"success": True,
|
|
"task_id": self.request.id,
|
|
"job_name": job_name,
|
|
"jenkins_url": jenkins_url,
|
|
"build_number": result.get("build_number"),
|
|
"build_url": result.get("build_url"),
|
|
"event_id": event_id,
|
|
"repository": repository,
|
|
"branch": branch,
|
|
"commit_hash": commit_hash,
|
|
"execution_time": execution_time,
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
}
|
|
else:
|
|
logger.error("Jenkins job trigger failed",
|
|
task_id=self.request.id,
|
|
job_name=job_name,
|
|
error=result.get("error"),
|
|
execution_time=execution_time)
|
|
|
|
# Retry task
|
|
raise self.retry(
|
|
countdown=settings.queue.retry_delay * (2 ** self.request.retries),
|
|
max_retries=settings.queue.max_retries
|
|
)
|
|
|
|
except Exception as e:
|
|
execution_time = time.time() - start_time
|
|
logger.error("Unexpected error in Jenkins task",
|
|
task_id=self.request.id,
|
|
job_name=job_name,
|
|
error=str(e),
|
|
execution_time=execution_time)
|
|
|
|
# Retry task
|
|
raise self.retry(
|
|
countdown=settings.queue.retry_delay * (2 ** self.request.retries),
|
|
max_retries=settings.queue.max_retries
|
|
)
|
|
|
|
|
|
@celery_app.task(
|
|
bind=True,
|
|
base=JenkinsTask,
|
|
max_retries=2,
|
|
default_retry_delay=30
|
|
)
|
|
def check_jenkins_health(
|
|
self,
|
|
jenkins_url: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Check Jenkins health status
|
|
Args:
|
|
jenkins_url: Jenkins URL
|
|
Returns:
|
|
Dict: health check result
|
|
"""
|
|
try:
|
|
logger.info("Checking Jenkins health", jenkins_url=jenkins_url)
|
|
|
|
jenkins_service = JenkinsService()
|
|
result = asyncio.run(jenkins_service.check_health(jenkins_url))
|
|
|
|
return {
|
|
"success": True,
|
|
"jenkins_url": jenkins_url,
|
|
"healthy": result.get("healthy", False),
|
|
"response_time": result.get("response_time"),
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Jenkins health check failed",
|
|
jenkins_url=jenkins_url,
|
|
error=str(e))
|
|
|
|
return {
|
|
"success": False,
|
|
"jenkins_url": jenkins_url,
|
|
"error": str(e),
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
}
|
|
|
|
|
|
@celery_app.task(
|
|
bind=True,
|
|
base=JenkinsTask
|
|
)
|
|
def cleanup_expired_tasks(self) -> Dict[str, Any]:
|
|
"""
|
|
Clean up expired tasks
|
|
Returns:
|
|
Dict: cleanup result
|
|
"""
|
|
try:
|
|
logger.info("Starting task cleanup")
|
|
|
|
# Get all tasks
|
|
inspect = self.app.control.inspect()
|
|
|
|
# Clean up expired results
|
|
cleaned_count = 0
|
|
current_time = time.time()
|
|
|
|
# Add more complex cleanup logic here if needed
|
|
# For example, clean up results older than a certain time
|
|
|
|
logger.info("Task cleanup completed", cleaned_count=cleaned_count)
|
|
|
|
return {
|
|
"success": True,
|
|
"cleaned_count": cleaned_count,
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Task cleanup failed", error=str(e))
|
|
|
|
return {
|
|
"success": False,
|
|
"error": str(e),
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
}
|
|
|
|
|
|
# Periodic tasks
|
|
@celery_app.on_after_configure.connect
|
|
def setup_periodic_tasks(sender, **kwargs):
|
|
"""Set up periodic tasks"""
|
|
|
|
# Clean up expired tasks every hour
|
|
sender.add_periodic_task(
|
|
3600.0, # 1 hour
|
|
cleanup_expired_tasks.s(),
|
|
name="cleanup-expired-tasks"
|
|
)
|
|
|
|
# Check Jenkins health every 5 minutes
|
|
for env_name, env_config in settings.environments.items():
|
|
sender.add_periodic_task(
|
|
300.0, # 5 minutes
|
|
check_jenkins_health.s(env_config.jenkins_url),
|
|
name=f"check-jenkins-health-{env_name}"
|
|
)
|
|
|
|
|
|
def get_celery_app() -> Celery:
|
|
"""Get Celery app instance"""
|
|
return celery_app |