Merge pull request #45 from CandoImage/prometheus-plugin

add generic prometheus plugin

Author: Rockstar04

README.rst

@@ -15,6 +15,7 @@ NewRelic platform. Currently supported backend systems are:
 - pgBouncer
 - PHP FPM
 - PostgreSQL
+- Prometheus
 - RabbitMQ
 - Redis
 - Riak
@@ -80,7 +81,7 @@ On Ubuntu systems 16.04 and newer, you can try running ``systemctl enable newrel
 Installing Additional Requirements
 ----------------------------------
 
-To use the MongoDB the ``mongodb`` library is required. For the pgBouncer or PostgreSQL plugin you must install the ``psycopg2`` library. To easily do
+To use the MongoDB the ``mongodb`` library is required. For the pgBouncer or PostgreSQL plugin you must install the ``psycopg2`` library. For the prometheus plugin you must install the ``prometheus_client`` library. To easily do
 this, make sure you have the latest version of ``pip`` installed (https://pip.pypa.io/). This should be done after installing the agent itself:
 
 ::
@@ -95,6 +96,10 @@ or::
 
     $ pip install newrelic-python-agent[postgresql]
 
+or::
+
+    $ pip install newrelic-python-agent[prometheus]
+
 If this does not work for you, make sure you are running a recent copy of ``pip`` (>= 1.3).
 
 Plugin Configuration Stanzas
@@ -313,6 +318,46 @@ E.g.:
       superuser: False
       relation_stats: False
 
+Prometheus Installation Notes
+------------------------
+You can monitor any prometheus exporter endpoint via http/https.
+Prometheus samples are named by their name and their labels.
+E.g. ``http_requests_total{job="apiserver", status=200}`` becomes ``http_requests_total/job/apiserver/status/200``
+
+By default all samples are added as derive values. To treat a sample as gauge value you can use the ``gauges`` configuration value.
+E.g.:
+
+::
+
+  prometheus:
+     -  name: my-go-app
+        scheme: http
+        host: localhost
+        port: 8080
+        gauges:
+          - go_threads
+
+Furthermore the plugin allows you to include and exclude certain samples by their name.
+If the ``include`` configuration parameter is not set, all samples are included (except for excluded ones).
+If the ``include`` configuration parameter is set, only those samples are included.
+If the ``exclude`` configuration parameter is set, those samples are excluded (even if they are listed in the ``include`` configuration parameter).
+E.g.:
+
+::
+
+  prometheus:
+     -  name: my-go-app
+        scheme: http
+        host: localhost
+        port: 8080
+        include:
+          - go_threads
+          - go_info
+        exclude:
+          - go_info
+
+If you are monitoring a prometheus exporter via a HTTPS connection you can use the ``verify_ssl_cert`` configuration value to disable SSL certificate verification.
+
 RabbitMQ Installation Notes
 ---------------------------
 The user specified must have access to all virtual hosts you wish to monitor and should have either the Administrator tag or the Monitoring tag.
@@ -462,6 +507,14 @@ Configuration Example
           dbname: postgres
           superuser: True
 
+      prometheus:
+         -  name: my-go-app
+            scheme: http
+            host: localhost
+            port: 8080
+            gauges:
+              - go_threads
+
       rabbitmq:
         - name: rabbitmq@localhost
           host: localhost

newrelic_python_agent/plugins/__init__.py

@@ -19,6 +19,7 @@
     'php_apc': 'newrelic_python_agent.plugins.php_apc.APC',
     'php_fpm': 'newrelic_python_agent.plugins.php_fpm.FPM',
     'postgresql': 'newrelic_python_agent.plugins.postgresql.PostgreSQL',
+    'prometheus': 'newrelic_python_agent.plugins.prometheus.Prometheus',
     'rabbitmq': 'newrelic_python_agent.plugins.rabbitmq.RabbitMQ',
     'redis': 'newrelic_python_agent.plugins.redis.Redis',
     'riak': 'newrelic_python_agent.plugins.riak.Riak',

newrelic_python_agent/plugins/prometheus.py

@@ -0,0 +1,65 @@
+"""
+Generic prometheus Support
+
+"""
+from prometheus_client.parser import text_string_to_metric_families
+from functools import reduce
+import logging
+import re
+
+from newrelic_python_agent.plugins import base
+
+LOGGER = logging.getLogger(__name__)
+
+class Prometheus(base.HTTPStatsPlugin):
+
+    DEFAULT_PATH = 'metrics'
+    GUID = 'com.meetme.newrelic_prometheus_agent'
+    INCLUDE_CONFIG_KEY = 'include'
+    EXCLUDE_CONFIG_KEY = 'exclude'
+    GAUGES_CONFIG_KEY = 'gauges'
+
+    def __init__(self, config, poll_interval, last_interval_values=None):
+        super(Prometheus, self).__init__(config, poll_interval, last_interval_values)
+
+
+    def add_datapoints(self, raw_metrics):
+        """Add all of the data points for a node
+
+        :param str metrics: The metrics content
+
+        """
+        hasMetrics = False
+        if not raw_metrics:
+            return
+        for family in text_string_to_metric_families(raw_metrics):
+            for sample in family.samples:
+                hasMetrics = True
+                if (
+                    not self.INCLUDE_CONFIG_KEY in self.config or
+                    sample.name in self.config[self.INCLUDE_CONFIG_KEY]
+                ):
+                    if (
+                        self.EXCLUDE_CONFIG_KEY in self.config and
+                        sample.name in self.config[self.EXCLUDE_CONFIG_KEY]
+                    ):
+                        LOGGER.debug('Ignoring sample: %r', sample)
+                    else:
+                        name = reduce(
+                            (lambda k, i: k + '/' + i[0] + '/' + i[1]),
+                            sample.labels.iteritems(),
+                            sample.name
+                        )
+                        if (
+                            self.GAUGES_CONFIG_KEY in self.config and
+                            sample.name in self.config[self.GAUGES_CONFIG_KEY]
+                        ):
+                            self.add_gauge_value(name,
+                                sample.name,
+                                sample.value)
+                        else:
+                            self.add_derive_value(name,
+                                sample.name,
+                                sample.value)
+        if not hasMetrics:
+            LOGGER.debug('Metrics output: %r', raw_metrics)

setup.py

@@ -16,7 +16,8 @@
 tests_require = []
 extras_require = {'mongodb': ['pymongo'],
                   'pgbouncer': ['psycopg2'],
-                  'postgresql': ['psycopg2']}
+                  'postgresql': ['psycopg2'],
+                  'prometheus': ['prometheus_client']}
 
 if sys.version_info < (2, 7, 0):
     install_requires.append('importlib')