Monitoring with Prometheus - Instrument Project API

Author: ChristopherGS

packages/ml_api/Makefile

@@ -25,6 +25,10 @@ run-service-development:
 	@echo "+ $@"
 	python run.py
 
+run-service-wsgi:
+	@echo "+ $@"
+	gunicorn --workers=1 --bind 0.0.0.0:5000 run:application
+
 db-migrations:
 	@echo "+ $@"
 	PYTHONPATH=. alembic -c alembic.ini upgrade head

packages/ml_api/api/app.py

@@ -4,6 +4,7 @@
 from sqlalchemy.orm import scoped_session
 
 from api.config import Config
+from api.monitoring.middleware import setup_metrics
 from api.persistence.core import init_database
 
 _logger = logging.getLogger(__name__)
@@ -23,6 +24,9 @@ def create_app(
     # Setup database
     init_database(flask_app, config=config_object, db_session=db_session)
 
+    # Setup prometheus monitoring
+    setup_metrics(flask_app)
+
     connexion_app.add_api("api.yaml")
     _logger.info("Application instance created")
 

packages/ml_api/api/config.py

@@ -14,6 +14,8 @@
 # Project Directories
 ROOT = pathlib.Path(api.__file__).resolve().parent.parent
 
+APP_NAME = 'ml_api'
+
 
 class Config:
     DEBUG = False

packages/ml_api/api/monitoring/__init__.py

@@ -0,0 +1,60 @@
+from flask import request, Flask
+from flask.wrappers import Response
+from prometheus_client import Counter, Histogram
+import time
+
+from api.config import APP_NAME
+
+
+# Counter and Histogram are examples of default metrics
+# available from the prometheus Python client.
+REQUEST_COUNT = Counter(
+    name='http_request_count',
+    documentation='App Request Count',
+    labelnames=['app_name', 'method', 'endpoint', 'http_status']
+)
+REQUEST_LATENCY = Histogram(
+    name='http_request_latency_seconds',
+    documentation='Request latency',
+    labelnames=['app_name', 'endpoint']
+)
+
+
+def start_timer() -> None:
+    """Get start time of a request."""
+    request._prometheus_metrics_request_start_time = time.time()
+
+
+def stop_timer(response: Response) -> Response:
+    """Get stop time of a request.."""
+    request_latency = time.time() - request._prometheus_metrics_request_start_time
+    REQUEST_LATENCY.labels(
+        app_name=APP_NAME,
+        endpoint=request.path).observe(request_latency)
+    return response
+
+
+def record_request_data(response: Response) -> Response:
+    """Capture request data.
+
+    Uses the flask request object to extract information such as
+    the HTTP request method, endpoint and HTTP status.
+    """
+    REQUEST_COUNT.labels(
+        app_name=APP_NAME,
+        method=request.method,
+        endpoint=request.path,
+        http_status=response.status_code).inc()
+    return response
+
+
+def setup_metrics(app: Flask) -> None:
+    """Setup Prometheus metrics.
+
+    This function uses the flask before_request
+    and after_request hooks to capture metrics
+    with each HTTP request to the application.
+    """
+    app.before_request(start_timer)
+    app.after_request(record_request_data)
+    app.after_request(stop_timer)