""" Configuration Celery pour le worker Notariat Pipeline """ import os from celery import Celery from kombu import Queue # Configuration Redis REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") # Création de l'application Celery app = Celery( 'notariat_worker', broker=REDIS_URL, backend=REDIS_URL, include=[ 'services.worker.tasks.pipeline_tasks', 'services.worker.tasks.ocr_tasks', 'services.worker.tasks.classification_tasks', 'services.worker.tasks.extraction_tasks', 'services.worker.tasks.indexing_tasks', 'services.worker.tasks.verification_tasks' ] ) # Configuration Celery app.conf.update( # Configuration des queues task_routes={ 'services.worker.tasks.pipeline_tasks.*': {'queue': 'pipeline'}, 'services.worker.tasks.ocr_tasks.*': {'queue': 'ocr'}, 'services.worker.tasks.classification_tasks.*': {'queue': 'classification'}, 'services.worker.tasks.extraction_tasks.*': {'queue': 'extraction'}, 'services.worker.tasks.indexing_tasks.*': {'queue': 'indexing'}, 'services.worker.tasks.verification_tasks.*': {'queue': 'verification'}, }, # Configuration des queues task_default_queue='default', task_queues=( Queue('default', routing_key='default'), Queue('pipeline', routing_key='pipeline'), Queue('ocr', routing_key='ocr'), Queue('classification', routing_key='classification'), Queue('extraction', routing_key='extraction'), Queue('indexing', routing_key='indexing'), Queue('verification', routing_key='verification'), ), # Configuration des tâches task_serializer='json', accept_content=['json'], result_serializer='json', timezone='Europe/Paris', enable_utc=True, # Configuration de la concurrence worker_concurrency=int(os.getenv("WORKER_CONCURRENCY", "2")), worker_prefetch_multiplier=1, task_acks_late=True, worker_disable_rate_limits=False, # Configuration des timeouts task_soft_time_limit=300, # 5 minutes task_time_limit=600, # 10 minutes worker_max_tasks_per_child=1000, # Configuration des retry task_default_retry_delay=60, task_max_retries=3, # Configuration des résultats result_expires=3600, # 1 heure result_persistent=True, # Configuration du monitoring worker_send_task_events=True, task_send_sent_event=True, # Configuration des logs worker_log_format='[%(asctime)s: %(levelname)s/%(processName)s] %(message)s', worker_task_log_format='[%(asctime)s: %(levelname)s/%(processName)s][%(task_name)s(%(task_id)s)] %(message)s', ) # Configuration des tâches périodiques (Celery Beat) app.conf.beat_schedule = { 'health-check': { 'task': 'services.worker.tasks.pipeline_tasks.health_check', 'schedule': 60.0, # Toutes les minutes }, 'cleanup-old-results': { 'task': 'services.worker.tasks.pipeline_tasks.cleanup_old_results', 'schedule': 3600.0, # Toutes les heures }, 'update-external-data': { 'task': 'services.worker.tasks.verification_tasks.update_external_data', 'schedule': 86400.0, # Tous les jours }, } # Configuration des signaux @app.task(bind=True) def debug_task(self): """Tâche de debug pour tester Celery""" print(f'Request: {self.request!r}') return 'Debug task completed' if __name__ == '__main__': app.start()