from celery import Celery from worker_app.core.config import get_settings settings = get_settings() celery_app = Celery(settings.worker_name) celery_app.conf.broker_url = settings.broker_url celery_app.conf.result_backend = settings.result_backend celery_app.conf.broker_connection_retry_on_startup = True # #1714 队列隔离:generation(高优,独占 worker)/ transcode(素材转码)/ celery(默认) from packages.shared.celery_queues import ( # noqa: E402 GENERATION_WORKER_PREFETCH_MULTIPLIER, apply_queue_settings, ) apply_queue_settings(celery_app) # 长渲染任务预取 1,避免任务被预取占住导致调度不均 celery_app.conf.worker_prefetch_multiplier = GENERATION_WORKER_PREFETCH_MULTIPLIER celery_app.conf.task_acks_late = True # worker 崩溃时未完成任务重回队列,由执行前守卫丢弃作废消息 # worker 进程被 OOM/容器硬杀时拒绝 ack,消息留在队列由其他 worker 接手 celery_app.conf.task_reject_on_worker_lost = True # Redis broker 消息可见性超时(#1714):acks_late 下,消息被预取后 visibility_timeout # 内未 ack 才会重投。长任务(ingest HEVC 转码 20-30 分钟、生成硬超时 11 分钟) # 必须远大于最长执行时间,否则正常任务会在执行中被误重投;4 小时覆盖最长转码 + 余量。 celery_app.conf.broker_transport_options = {"visibility_timeout": 4 * 60 * 60} celery_app.conf.imports = ( "worker_app.tasks.health", "worker_app.tasks.ingest", "worker_app.tasks.classification", "worker_app.tasks.generation", "worker_app.tasks.voice_extraction", "worker_app.tasks.voice_clone", "worker_app.tasks.tts_synthesis", "worker_app.tasks.batch_download", "worker_app.tasks.duplication_check", "worker_app.tasks._startup", "apps.worker.video_processing.dedup", "worker_app.tasks.cleanup", ) # Celery Beat 定时任务调度 # 注:worker 单实例内嵌 beat(entrypoint-worker.sh -B),定时任务不会重复执行 celery_app.conf.beat_schedule = { # pending 任务超时清理:worker 停止消费后,卡 pending 的任务 15 分钟内释放限流名额 "cleanup-stale-pending-tasks": { "task": "worker.cleanup_stale_pending_tasks", "schedule": 300.0, # 每 5 分钟(秒) "options": {"expires": 240}, # 4 分钟过期,避免堆积 }, # running 孤儿任务巡检:容器重启/进程被杀后卡 running 的任务,20 分钟无更新则判失败 "cleanup-stale-running-tasks": { "task": "worker.cleanup_stale_running_tasks", "schedule": 300.0, # 每 5 分钟(秒) "options": {"expires": 240}, }, # 上传/转码链路孤儿巡检:worker 重启丢 prefetch 消息后,卡 pending/processing # 的 ingest_job + asset 占位超时标终态(#1714)。转码任务较长,10 分钟一轮 "cleanup-stale-ingest-jobs": { "task": "worker.cleanup_stale_ingest_jobs", "schedule": 600.0, # 每 10 分钟(秒) "options": {"expires": 540}, }, }