export RQ status as prometheus metrics

fixes: #503

Author: terencehonles

django_rq/apps.py

@@ -1,6 +1,48 @@
 from django.apps import AppConfig
 
+from .queues import filter_connection_params, get_connection, get_queue_class, get_unique_connection_configs
+from .workers import get_worker_class
+
+import logging
+
+log = logging.getLogger(__name__)
+
+try:
+    import prometheus_client
+
+    from rq_exporter.collector import RQCollector
+except ImportError:
+    RQCollector = None
+
 
 class DjangoRqAdminConfig(AppConfig):
     default_auto_field = "django.db.models.AutoField"
     name = "django_rq"
+
+    def ready(self):
+        if RQCollector is None:
+            return
+
+        from .settings import QUEUES
+
+        worker_class = get_worker_class()
+        collector_args = {}
+        collector_counts = {}
+        unique_configs = get_unique_connection_configs()
+        for name, config in QUEUES.items():
+            index = unique_configs.index(filter_connection_params(config))
+            queue_class = get_queue_class(config)
+            key = index, queue_class
+            if key not in collector_args:
+                collector_args[key] = [name, worker_class, queue_class]
+                collector_counts[key] = 1
+            else:
+                collector_counts[key] += 1
+
+        if len(collector_args) > 1:
+            log.warning('RQCollector can only log metrics for one unique connection and queue class')
+
+        if collector_args:
+            key = sorted(collector_counts.items(), key=lambda x: x[1], reverse=True)[0][0]
+            name, worker_class, queue_class = collector_args[key]
+            prometheus_client.REGISTRY.register(RQCollector(get_connection(name), worker_class, queue_class))

django_rq/urls.py

@@ -2,9 +2,19 @@
 
 from . import views
 
+try:
+    import rq_exporter
+
+    metrics_view = [
+        re_path(r'^metrics/?$', views.prometheus_metrics, name='rq_metrics'),
+    ]
+except ImportError:
+    metrics_view = []
+
 urlpatterns = [
     re_path(r'^$', views.stats, name='rq_home'),
     re_path(r'^stats.json/(?P<token>[\w]+)?/?$', views.stats_json, name='rq_home_json'),
+    *metrics_view,
     re_path(r'^queues/(?P<queue_index>[\d]+)/$', views.jobs, name='rq_jobs'),
     re_path(r'^workers/(?P<queue_index>[\d]+)/$', views.workers, name='rq_workers'),
     re_path(r'^workers/(?P<queue_index>[\d]+)/(?P<key>[-\w\.\:\$]+)/$', views.worker_details, name='rq_worker_details'),

django_rq/views.py

@@ -4,7 +4,7 @@
 
 from django.contrib import admin, messages
 from django.contrib.admin.views.decorators import staff_member_required
-from django.http import Http404, JsonResponse
+from django.http import Http404, HttpResponse, JsonResponse
 from django.shortcuts import redirect, render
 from django.urls import reverse
 from django.views.decorators.cache import never_cache
@@ -27,6 +27,11 @@
 from .settings import API_TOKEN, QUEUES_MAP
 from .utils import get_jobs, get_scheduler_statistics, get_statistics, stop_jobs
 
+try:
+    import prometheus_client
+except ImportError:
+    prometheus_client = None
+
 
 @never_cache
 @staff_member_required
@@ -48,6 +53,20 @@ def stats_json(request, token=None):
     )
 
 
+@never_cache
+@staff_member_required
+def prometheus_metrics(request):
+    if not prometheus_client:
+        raise Http404
+
+    registry = prometheus_client.REGISTRY
+    encoder, content_type = prometheus_client.exposition.choose_encoder(request.META.get('HTTP_ACCEPT', ''))
+    if 'name[]' in request.GET:
+        registry = registry.restricted_registry(request.GET.getlist('name[]'))
+
+    return HttpResponse(encoder(registry), headers={'Content-Type': content_type})
+
+
 @never_cache
 @staff_member_required
 def jobs(request, queue_index):

setup.py

@@ -16,6 +16,7 @@
     package_data={'': ['README.rst']},
     install_requires=['django>=3.2', 'rq>=1.14', 'redis>=3'],
     extras_require={
+        'prometheus-metrics': ['prometheus_client>=0.4.0', 'rq-exporter'],
         'Sentry': ['sentry-sdk>=1.0.0'],
         'testing': [],
     },