feat: decoupled prometheus exporter's calculation and output (#12383)

Author: SkyeYoung

apisix/cli/config.lua

@@ -95,6 +95,7 @@ local _M = {
     meta = {
       lua_shared_dict = {
         ["prometheus-metrics"] = "15m",
+        ["prometheus-cache"] = "10m",
         ["standalone-config"] = "10m",
         ["status-report"] = "1m",
       }
@@ -323,7 +324,8 @@ local _M = {
       export_addr = {
         ip = "127.0.0.1",
         port = 9091
-      }
+      },
+      refresh_interval = 15
     },
     ["server-info"] = {
       report_ttl = 60

apisix/cli/ngx_tpl.lua

@@ -67,6 +67,9 @@ lua {
     {% if enabled_stream_plugins["prometheus"] then %}
     lua_shared_dict prometheus-metrics {* meta.lua_shared_dict["prometheus-metrics"] *};
     {% end %}
+    {% if enabled_plugins["prometheus"] or enabled_stream_plugins["prometheus"] then %}
+    lua_shared_dict prometheus-cache {* meta.lua_shared_dict["prometheus-cache"] *};
+    {% end %}
     {% if standalone_with_admin_api then %}
     lua_shared_dict standalone-config {* meta.lua_shared_dict["standalone-config"] *};
     {% end %}
@@ -96,22 +99,20 @@ http {
     }
 
     init_worker_by_lua_block {
-        require("apisix.plugins.prometheus.exporter").http_init(true)
+        local prometheus = require("apisix.plugins.prometheus.exporter")
+        prometheus.http_init(true)
+        prometheus.init_exporter_timer()
     }
 
     server {
-        {% if use_apisix_base then %}
-            listen {* prometheus_server_addr *} enable_process=privileged_agent;
-        {% else %}
-            listen {* prometheus_server_addr *};
-        {% end %}
+        listen {* prometheus_server_addr *};
 
         access_log off;
 
         location / {
             content_by_lua_block {
                 local prometheus = require("apisix.plugins.prometheus.exporter")
-                prometheus.export_metrics(true)
+                prometheus.export_metrics()
             }
         }
 
@@ -577,11 +578,7 @@ http {
 
     {% if enabled_plugins["prometheus"] and prometheus_server_addr then %}
     server {
-        {% if use_apisix_base then %}
-            listen {* prometheus_server_addr *} enable_process=privileged_agent;
-        {% else %}
-            listen {* prometheus_server_addr *};
-        {% end %}
+        listen {* prometheus_server_addr *};
 
         access_log off;
 

apisix/core/config_etcd.lua

@@ -1001,7 +1001,7 @@ function _M.new(key, opts)
         sync_times = 0,
         running = true,
         conf_version = 0,
-        values = nil,
+        values = {},
         need_reload = true,
         watching_stream = nil,
         routes_hash = nil,

apisix/init.lua

@@ -155,6 +155,10 @@ function _M.http_init_worker()
     if local_conf.apisix and local_conf.apisix.enable_server_tokens == false then
         ver_header = "APISIX"
     end
+
+    -- To ensure that all workers related to Prometheus metrics are initialized,
+    -- we need to put the initialization of the Prometheus plugin here.
+    plugin.init_prometheus()
 end
 
 

apisix/plugin.lua

@@ -341,8 +341,6 @@ function _M.load(config)
         return local_plugins
     end
 
-    local exporter = require("apisix.plugins.prometheus.exporter")
-
     if ngx.config.subsystem == "http" then
         if not http_plugin_names then
             core.log.error("failed to read plugin list from local file")
@@ -356,15 +354,6 @@ function _M.load(config)
             if not ok then
                 core.log.error("failed to load plugins: ", err)
             end
-
-            local enabled = core.table.array_find(http_plugin_names, "prometheus") ~= nil
-            local active  = exporter.get_prometheus() ~= nil
-            if not enabled then
-                exporter.destroy()
-            end
-            if enabled and not active then
-                exporter.http_init()
-            end
         end
     end
 
@@ -808,18 +797,21 @@ do
 end
 
 
-function _M.init_worker()
+function _M.init_prometheus()
     local _, http_plugin_names, stream_plugin_names = get_plugin_names()
+    local enabled_in_http = core.table.array_find(http_plugin_names, "prometheus")
+    local enabled_in_stream = core.table.array_find(stream_plugin_names, "prometheus")
 
-    -- some plugins need to be initialized in init* phases
-    if is_http and core.table.array_find(http_plugin_names, "prometheus") then
-        local prometheus_enabled_in_stream =
-            core.table.array_find(stream_plugin_names, "prometheus")
-        require("apisix.plugins.prometheus.exporter").http_init(prometheus_enabled_in_stream)
-    elseif not is_http and core.table.array_find(stream_plugin_names, "prometheus") then
-        require("apisix.plugins.prometheus.exporter").stream_init()
+    -- For stream-only mode, there are separate calls in ngx_tpl.lua.
+    -- And for other modes, whether in stream or http plugins,
+    -- the prometheus exporter needs to be initialized.
+    if is_http and (enabled_in_http or enabled_in_stream) then
+        require("apisix.plugins.prometheus.exporter").init_exporter_timer()
     end
+end
 
+
+function _M.init_worker()
     -- someone's plugin needs to be initialized after prometheus
     -- see https://github.com/apache/apisix/issues/3286
     _M.load()

apisix/plugins/prometheus.lua

@@ -17,7 +17,6 @@
 local core = require("apisix.core")
 local exporter = require("apisix.plugins.prometheus.exporter")
 
-
 local plugin_name = "prometheus"
 local schema = {
     type = "object",
@@ -35,6 +34,7 @@ local _M = {
     priority = 500,
     name = plugin_name,
     log  = exporter.http_log,
+    destroy = exporter.destroy,
     schema = schema,
     run_policy = "prefer_route",
 }
@@ -55,4 +55,11 @@ function _M.api()
 end
 
 
+function _M.init()
+    local local_conf = core.config.local_conf()
+    local enabled_in_stream = core.table.array_find(local_conf.stream_plugins, "prometheus")
+    exporter.http_init(enabled_in_stream)
+end
+
+
 return _M