If REST port not accessible, try failing over to local port.

Author: russgold

README.md

@@ -57,7 +57,7 @@ Note that there are two parts to the configuration. The optional top portion def
 | `query_sync.interval` | The interval, in seconds, at which the service will be queried. Defaults to 10. |
 | `metricsNameSnakeCase` | If true, metrics names will be converted to snake case. Defaults to false. |
 | `domainQualifier` | If true, the domain name will be included as a qualifier for all metrics. Defaults to false. |
-| `restPort` | Optional. Overrides the port on which the exporter should contact the REST API. Needed when behind a load balancer. |
+| `restPort` | Optional. Overrides the port on which the exporter should contact the REST API. Needed if the exporter cannot find the REST API. |
 
 The `query` field is more complex. Each query consists of a hierarchy of the [MBeans](https://docs.oracle.com/middleware/1221/wls/WLMBR/core/index.html), starting relative to `ServerRuntimes`.
 Within each section, there are a number of options:
@@ -134,4 +134,4 @@ include the `restPort` configuration to tell the exporter which port to use.
 
 ## Copyright
 
- Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates. All rights reserved.
+ Copyright © 2017, 2020, Oracle Corporation and/or its affiliates.

src/main/java/io/prometheus/wls/rest/ConfigurationServlet.java

@@ -48,12 +48,15 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
     }
 
     private void updateConfiguration(WebClient webClient, HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
-        authenticate(webClient.withUrl(LiveConfiguration.getAuthenticationUrl(getProtocol())));
         try {
+            authenticate(webClient.withUrl(getAuthenticationUrl()));
             if (!ServletFileUpload.isMultipartContent(req)) throw new ServletException("Must be a multi-part request");
 
             createPostAction(req).perform();
             reportUpdatedConfiguration(resp);
+        } catch (RestPortConnectionException e) {
+            reportFailure(e);
+            webClient.setRetryNeeded(true);
         } catch (ConfigurationException e) {
             reportUnableToUpdateConfiguration(req, resp.getOutputStream(), e);
         }

src/main/java/io/prometheus/wls/rest/ExporterServlet.java

@@ -6,6 +6,8 @@
  */
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.Map;
 import java.util.TreeMap;
 import javax.servlet.ServletConfig;
@@ -48,19 +50,23 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se
     @SuppressWarnings("unused") // The req parameter is not used, but is required by 'doWithAuthentication'
     private void displayMetrics(WebClient webClient, HttpServletRequest req, HttpServletResponse resp) throws IOException {
         LiveConfiguration.updateConfiguration();
-        try (MetricsStream metricsStream = new MetricsStream(resp.getOutputStream())) {
+        try (MetricsStream metricsStream = new MetricsStream(req, resp.getOutputStream())) {
             if (!LiveConfiguration.hasQueries())
                 metricsStream.println("# No configuration defined.");
             else
                 printMetrics(webClient, metricsStream);
-
         }
     }
 
     private void printMetrics(WebClient webClient, MetricsStream metricsStream) throws IOException {
-        for (MBeanSelector selector : LiveConfiguration.getQueries())
-            displayMetrics(webClient, metricsStream, selector);
-        metricsStream.printPerformanceMetrics();
+        try {
+            for (MBeanSelector selector : LiveConfiguration.getQueries())
+                displayMetrics(webClient, metricsStream, selector);
+            metricsStream.printPerformanceMetrics();
+        } catch (RestPortConnectionException e) {
+            reportFailure(e);
+            webClient.setRetryNeeded(true);
+        }
     }
 
     private void displayMetrics(WebClient webClient, MetricsStream metricsStream, MBeanSelector selector) throws IOException {
@@ -73,9 +79,20 @@ private void displayMetrics(WebClient webClient, MetricsStream metricsStream, MB
                   withCommentMarkers("REST service was unable to handle this query\n"
                       + selector.getPrintableRequest() + '\n'
                       + "exception: " + e.getMessage()));
+        }  catch (IOException | RuntimeException | Error e) {
+            MessagesServlet.addExchange(getQueryUrl(selector), selector.getRequest(), toStackTrace(e));
+            throw e;
         }
     }
 
+    private String toStackTrace(Throwable e) {
+        StringWriter sw = new StringWriter();
+        PrintWriter pw = new PrintWriter(sw);
+        e.printStackTrace(pw);
+        pw.close();
+        return sw.toString();
+    }
+
     private String withCommentMarkers(String string) {
         StringBuilder sb = new StringBuilder();
         for (String s : string.split("\\r?\\n"))
@@ -95,7 +112,7 @@ private Map<String, Object> getMetrics(WebClient webClient, MBeanSelector select
     }
 
     private String requestMetrics(WebClient webClient, MBeanSelector selector) throws IOException {
-        String url = LiveConfiguration.getUrl(getProtocol(), selector);
+        String url = getQueryUrl(selector);
         String jsonResponse = webClient.withUrl(url).doPostRequest(selector.getRequest());
         MessagesServlet.addExchange(url, selector.getRequest(), jsonResponse);
         return jsonResponse;

src/main/java/io/prometheus/wls/rest/LiveConfiguration.java

@@ -13,14 +13,13 @@
 import java.util.Optional;
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
 
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 import io.prometheus.wls.rest.domain.ExporterConfig;
 import io.prometheus.wls.rest.domain.MBeanSelector;
-import io.prometheus.wls.rest.domain.Protocol;
 import io.prometheus.wls.rest.domain.QuerySyncConfiguration;
-import io.prometheus.wls.rest.domain.QueryType;
 import org.yaml.snakeyaml.Yaml;
 
 /**
@@ -79,35 +78,25 @@ static void setServer(String serverName, int serverPort) {
     }
 
     /**
-     * Returns the URL used to query the management services
-     * @return a url built for the configured server
-     * @param protocol the protocol used to authenticate a configuration change
+     * Specifies the server on which to contact the Management RESTful services.
+     *
+     * @param req the incoming request
      */
-    static String getAuthenticationUrl(Protocol protocol) {
-        return protocol.format(QueryType.RUNTIME_URL_PATTERN, WLS_HOST, getRestPort());
+    static void setServer(HttpServletRequest req) {
+        LiveConfiguration.setServer(req.getServerName(), req.getServerPort());
     }
 
     /**
-     * Returns the URL used to query the management services
-     *
-     * @param protocol the protocol to use
-     * @param selector the selector which will define the query
-     * @return a url built for the configured server
+     * Creates a builder for URLs that can handle retries to alternative ports.
+     * @param request the active servlet request
+     * @return the new builder
      */
-    static String getUrl(Protocol protocol, MBeanSelector selector) {
-        return selector.getUrl(protocol, WLS_HOST, getRestPort());
+    static UrlBuilder createUrlBuilder(HttpServletRequest request) {
+        return new UrlBuilder(request, getConfiguredRestPort());
     }
 
-    private static int getRestPort() {
-        return Optional.ofNullable(config.getRestPort()).orElse(serverPort);
-    }
-
-    /**
-     * Returns the qualifiers to add to the performance metrics, specifying the configured server
-     * @return a metrics qualifier string
-     */
-    static String getPerformanceQualifier() {
-        return String.format("{instance=\"%s:%d\"}", serverName, serverPort);
+    static Integer getConfiguredRestPort() {
+        return Optional.ofNullable(config).map(ExporterConfig::getRestPort).orElse(null);
     }
 
     /**

src/main/java/io/prometheus/wls/rest/MainServlet.java

@@ -1,19 +1,22 @@
 package io.prometheus.wls.rest;
 /*
- * Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
  */
+import java.io.IOException;
+import java.io.PrintStream;
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletOutputStream;
 import javax.servlet.annotation.WebServlet;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.PrintStream;
 
-import static io.prometheus.wls.rest.ServletConstants.*;
+import static io.prometheus.wls.rest.ServletConstants.APPEND_ACTION;
+import static io.prometheus.wls.rest.ServletConstants.CONFIGURATION_PAGE;
+import static io.prometheus.wls.rest.ServletConstants.EFFECT_OPTION;
+import static io.prometheus.wls.rest.ServletConstants.REPLACE_ACTION;
 
 /**
  * This servlet represents the 'landing page' for the exporter.
@@ -30,8 +33,8 @@ public void init(ServletConfig servletConfig) {
 
     @Override
     protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
-        LiveConfiguration.setServer(req.getServerName(), req.getServerPort());
         LiveConfiguration.updateConfiguration();
+        LiveConfiguration.setServer(req);
         resp.getOutputStream().println(ServletConstants.PAGE_HEADER);
         displayMetricsLink(req, resp.getOutputStream());
         displayForm(req, resp.getOutputStream());

src/main/java/io/prometheus/wls/rest/MessagesServlet.java

@@ -18,14 +18,14 @@
 import static io.prometheus.wls.rest.ServletConstants.MESSAGES_PAGE;
 
 /**
- * A collector of REST requests and replies, that can be viewed to diagnose problems.List<String
+ * A collector of REST requests and replies, that can be viewed to diagnose problems.
  */
 @WebServlet("/" + MESSAGES_PAGE)
 public class MessagesServlet extends HttpServlet {
     static final int MAX_EXCHANGES = 5;
     private static final String TEMPLATE = "REQUEST to %s:%n%s%nREPLY:%n%s%n";
 
-    private static Queue<String> messages = new ConcurrentLinkedDeque<>();
+    private static final Queue<String> messages = new ConcurrentLinkedDeque<>();
 
     @Override
     protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {

src/main/java/io/prometheus/wls/rest/MetricsStream.java

@@ -1,17 +1,18 @@
 package io.prometheus.wls.rest;
 /*
- * Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
  */
-import com.sun.management.OperatingSystemMXBean;
-
-import javax.management.MBeanServerConnection;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.lang.management.ManagementFactory;
 import java.util.Locale;
+import javax.management.MBeanServerConnection;
+import javax.servlet.http.HttpServletRequest;
+
+import com.sun.management.OperatingSystemMXBean;
 
 /**
  * A PrintStream that computes metrics for the performance of the exporter itself. It does so by tracking the
@@ -22,6 +23,7 @@
 class MetricsStream extends PrintStream {
     private static final double NANOSEC_PER_SECONDS = 1000000000;
 
+    private final HttpServletRequest request;
     private final PerformanceProbe performanceProbe;
     private final long startTime;
     private final long startCpu;
@@ -30,20 +32,23 @@ class MetricsStream extends PrintStream {
 
     /**
      * Constructs a metrics stream object, installer a performance probe that access system data.
+     * @param request the request for the metrics
      * @param outputStream the parent output stream
      * @throws IOException if some error occurs while creating the performance probe
      */
-    MetricsStream(OutputStream outputStream) throws IOException {
-        this(outputStream, new PlatformPeformanceProbe());
+    MetricsStream(HttpServletRequest request, OutputStream outputStream) throws IOException {
+        this(request, outputStream, new PlatformPeformanceProbe());
     }
 
     /**
      * A constructor for unit testing, allowing the specification of a test version of the performance probe.
+     * @param request the request for the metrics
      * @param outputStream the parent output stream
      * @param performanceProbe an object which can return performance data
      */
-    MetricsStream(OutputStream outputStream, PerformanceProbe performanceProbe) {
+    MetricsStream(HttpServletRequest request, OutputStream outputStream, PerformanceProbe performanceProbe) {
         super(outputStream);
+        this.request = request;
         this.performanceProbe = performanceProbe;
         startTime = performanceProbe.getCurrentTime();
         startCpu = performanceProbe.getCurrentCpu();
@@ -69,15 +74,23 @@ void printPerformanceMetrics() {
     }
 
     private String getDurationName() {
-        return "wls_scrape_duration_seconds" + LiveConfiguration.getPerformanceQualifier();
+        return "wls_scrape_duration_seconds" + getPerformanceQualifier();
+    }
+
+    /**
+     * Returns the qualifiers to add to the performance metrics, specifying the configured server
+     * @return a metrics qualifier string
+     */
+    private String getPerformanceQualifier() {
+        return String.format("{instance=\"%s:%d\"}", request.getServerName(), request.getServerPort());
     }
 
     private String getCpuUsageName() {
-        return "wls_scrape_cpu_seconds" + LiveConfiguration.getPerformanceQualifier();
+        return "wls_scrape_cpu_seconds" + getPerformanceQualifier();
     }
 
     private String getCountName() {
-        return "wls_scrape_mbeans_count_total" + LiveConfiguration.getPerformanceQualifier();
+        return "wls_scrape_mbeans_count_total" + getPerformanceQualifier();
     }
 
     private long getElapsedTime() {

src/main/java/io/prometheus/wls/rest/PassThroughAuthenticationServlet.java

@@ -14,7 +14,8 @@
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
 
-import io.prometheus.wls.rest.domain.Protocol;
+import io.prometheus.wls.rest.domain.MBeanSelector;
+import io.prometheus.wls.rest.domain.QueryType;
 
 import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
@@ -26,19 +27,31 @@
  */
 abstract public class PassThroughAuthenticationServlet extends HttpServlet {
     private final WebClientFactory webClientFactory;
-    private Protocol protocol;
+    private UrlBuilder urlBuilder;
 
     PassThroughAuthenticationServlet(WebClientFactory webClientFactory) {
         this.webClientFactory = webClientFactory;
     }
 
-    private WebClient createWebClient(HttpServletRequest req) {
-        LiveConfiguration.setServer(req.getServerName(), req.getServerPort());
+    String getAuthenticationUrl() {
+        return urlBuilder.createUrl(QueryType.RUNTIME_URL_PATTERN);
+    }
+
+    String getQueryUrl(MBeanSelector selector) {
+        return urlBuilder.createUrl(selector.getQueryType().getUrlPattern());
+    }
+
+    void reportFailure(RestPortConnectionException e) {
+        urlBuilder.reportFailure(e);
+    }
+
+    WebClient createWebClient(HttpServletRequest req) {
+        LiveConfiguration.setServer(req);
+        urlBuilder = LiveConfiguration.createUrlBuilder(req);
         final WebClient webClient = webClientFactory.createClient();
         webClient.addHeader("X-Requested-By", "rest-exporter");
 
         forwardRequestHeaders(req, webClient);
-        protocol = Protocol.getProtocol(req);
         return webClient;
     }
 
@@ -71,8 +84,8 @@ private String getSessionCookie(HttpServletRequest req) {
     void doWithAuthentication(HttpServletRequest req, HttpServletResponse resp, AuthenticatedService authenticatedService) throws IOException, ServletException {
         try {
             WebClient webClient = createWebClient(req);
-            authenticatedService.execute(webClient, req, resp);
-            webClient.forwardResponseHeaders(resp);
+            performRequest(webClient, req, resp, authenticatedService);
+            urlBuilder.reportSuccess();
         } catch (ForbiddenException e) {
             resp.sendError(SC_FORBIDDEN, "Not authorized");
         } catch (AuthenticationChallengeException e) {
@@ -89,8 +102,11 @@ void doWithAuthentication(HttpServletRequest req, HttpServletResponse resp, Auth
         }
     }
 
-    protected Protocol getProtocol() {
-        return protocol;
+    private void performRequest(WebClient webClient, HttpServletRequest req, HttpServletResponse resp, AuthenticatedService authenticatedService) throws IOException, ServletException {
+        do {
+            authenticatedService.execute(webClient, req, resp);
+            webClient.forwardResponseHeaders(resp);
+        } while (webClient.isRetryNeeded());
     }
 
     private void reportUnableToContactRestApi(HttpServletResponse resp, String uri) throws IOException {