Merge pull request #6558 from psiinon/llm/stats

llm: add basic stats

Author: thc202

addOns/callhome/CHANGELOG.md

@@ -4,7 +4,8 @@ All notable changes to this add-on will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
-
+### Added
+- LLM stats to telemetry.
 
 ## [0.14.0] - 2025-01-09
 ### Changed

addOns/callhome/src/main/java/org/zaproxy/addon/callhome/ExtensionCallHome.java

@@ -299,6 +299,7 @@ public boolean test(Entry<String, Long> t) {
                     || key.startsWith("stats.fuzz.")
                     || key.startsWith("stats.graphql.")
                     || key.startsWith("stats.hud.")
+                    || key.startsWith("stats.llm.")
                     || key.startsWith("stats.network.")
                     || key.startsWith("stats.oast.")
                     || key.startsWith("stats.openapi.")

addOns/llm/CHANGELOG.md

@@ -6,4 +6,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## Unreleased
 ### Added
 - Allow to perform API sequencing and alert review.
-
+- Basic stats

addOns/llm/src/main/java/org/zaproxy/addon/llm/services/LlmAssistant.java

@@ -27,11 +27,11 @@
 
 public interface LlmAssistant {
     @UserMessage(
-            "Given the following OpenAPI definition, generate a list of chained HTTP requests to simulate a real world interaction : {{swagger}} ")
-    HttpRequestList extractHttpRequests(String swagger);
+            "Given the following OpenAPI definition, generate a list of chained HTTP requests to simulate a real world interaction : {{openapi}} ")
+    HttpRequestList extractHttpRequests(String openapi);
 
     @UserMessage(
-            "As a software architect, and based on your previous answer, generate other potential missing endpoints that are not mentioned in the swagger file. For example, if there is GET /product/1, suggest DELETE /product/1 if it's not mentioned")
+            "As a software architect, and based on your previous answer, generate other potential missing endpoints that are not mentioned in the OpenAPI file. For example, if there is GET /product/1, suggest DELETE /product/1 if it's not mentioned")
     HttpRequestList complete();
 
     @SystemMessage(

addOns/llm/src/main/java/org/zaproxy/addon/llm/services/LlmCommunicationService.java

@@ -48,6 +48,7 @@
 import org.zaproxy.addon.llm.utils.HistoryPersister;
 import org.zaproxy.addon.llm.utils.Requestor;
 import org.zaproxy.zap.extension.alert.ExtensionAlert;
+import org.zaproxy.zap.utils.Stats;
 
 public class LlmCommunicationService {
 
@@ -99,12 +100,15 @@ private ChatLanguageModel buildModel(LlmOptions options) {
         };
     }
 
-    private Integer importHttpCalls(String swaggercontent) throws IOException {
-        HttpRequestList listHttpRequest = llmAssistant.extractHttpRequests(swaggercontent);
+    private Integer importHttpCalls(String openapiContent) throws IOException {
+        Stats.incCounter("stats.llm.openapiseq.call");
+        HttpRequestList listHttpRequest = llmAssistant.extractHttpRequests(openapiContent);
         if (listHttpRequest == null) {
+            Stats.incCounter("stats.llm.openapiseq.fail");
             throw new RuntimeException(
                     "An issue occurred when trying to get response from the LLM");
         }
+        Stats.incCounter("stats.llm.openapiseq.result.count", listHttpRequest.getRequests().size());
         requestor.run(listHttpRequest);
         return listHttpRequest.getRequests().size();
     }
@@ -165,7 +169,15 @@ public void reviewAlert(Alert alert) {
             Confidence llmConfidence;
             LOGGER.debug("Reviewing alert : {}", alert.getName());
             LOGGER.debug("Confidence level from ZAP : {}", alert.getConfidence());
+            Stats.incCounter("stats.llm.alertreview.call");
             llmConfidence = llmAssistant.review(alert.getDescription(), alert.getEvidence());
+
+            if (llmConfidence.getLevel() == alert.getConfidence()) {
+                Stats.incCounter("stats.llm.alertreview.result.same");
+            } else {
+                Stats.incCounter("stats.llm.alertreview.result.changed");
+            }
+
             LOGGER.debug(
                     "Confidence level from LLM : {} | Explanation : {}",
                     llmConfidence.getLevel(),