feat(actors): expose container system metrics

Author: NathanFlurry

docker/dev-full/docker-compose.yml

@@ -233,6 +233,8 @@ services:
       # We only reserve 100 ports instead of the default 22,000. See
       # rivet-guard for explanation.
       - "7600-7699:7600-7699"
+      # cAdvisor metrics endpoint
+      - "7780:7780"
     networks:
       - rivet-network
 

docker/dev-full/otel-collector/config.yaml

@@ -5,6 +5,14 @@ receivers:
         endpoint: 0.0.0.0:4317
       http:
         endpoint: 0.0.0.0:4318
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'cadvisor'
+          static_configs:
+            - targets: ['rivet-client:7780']
+          metrics_path: /metrics
+          scrape_interval: 30s
 
 processors:
   batch:
@@ -52,7 +60,7 @@ service:
       processors: [batch]
       exporters: [clickhouse]
     metrics:
-      receivers: [otlp]
+      receivers: [otlp, prometheus]
       processors: [batch]
       exporters: [clickhouse]
 

docker/universal/Dockerfile

@@ -120,8 +120,8 @@ RUN apt-get update -y && \
     apt-get install -y ca-certificates openssl curl tini curl && \
     curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/7.1.60/libfdb_c.x86_64.so"
 
-# MARK: Runner (Full)
-FROM --platform=linux/amd64 base-runner AS client-full
+# MARK: Runner (Slim)
+FROM --platform=linux/amd64 base-runner AS client-slim
 ARG CNI_PLUGINS_VERSION=1.3.0
 RUN apt-get install -y skopeo iproute2 runc dnsutils && \
 	echo "Downloading lz4" && \
@@ -142,6 +142,21 @@ COPY ./docker/dev-full/rivet-client/rivet-actor.conflist /opt/cni/config/rivet-a
 COPY --from=builder /app/dist/rivet-client /app/dist/rivet-container-runner /usr/local/bin/
 ENTRYPOINT ["/usr/bin/tini", "--", "entrypoint.sh"]
 
+# MARK: Runner (Full)
+FROM client-slim AS client-full
+ARG CADVISOR_VERSION=v0.52.0
+RUN apt-get update -y && \
+    apt-get install -y wget && \
+    wget -O /usr/local/bin/cadvisor "https://github.com/google/cadvisor/releases/download/${CADVISOR_VERSION}/cadvisor-${CADVISOR_VERSION}-linux-amd64" && \
+    chmod +x /usr/local/bin/cadvisor && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY docker/universal/client-full-entrypoint.sh /usr/local/bin/client-full-entrypoint.sh
+RUN chmod +x /usr/local/bin/client-full-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/client-full-entrypoint.sh"]
+
 # MARK: Monlith
 FROM --platform=linux/amd64 debian:12.9-slim AS monolith
 ENV DEBIAN_FRONTEND=noninteractive

docker/universal/client-full-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+# Start cadvisor in the background
+cadvisor \
+  --port=7780 \
+  --listen_ip=0.0.0.0 \
+  --prometheus_endpoint="/metrics" \
+  --enable_metrics=cpu,cpuLoad,memory,network,disk,diskIO,oom_event,process,tcp,udp \
+  --docker_only=false \
+  --disable_root_cgroup_stats=false &
+
+# TODO:
+# --raw_cgroup_prefix_whitelist="" \
+
+# Start rivet-client with all passed arguments
+exec rivet-client "$@"

examples/system-test-actor/src/shared/server.ts

@@ -7,14 +7,14 @@ export function createAndStartServer(
 	getUpgradeWebSocket: GetUpgradeWebSocketFn,
 ): { app: Hono; port: number } {
 	// Setup auto-exit timer
-	setTimeout(() => {
-		console.error(
-			"Actor should've been destroyed by now. Automatically exiting.",
-		);
-
-		if (typeof Deno !== "undefined") Deno.exit(1);
-		else process.exit(1);
-	}, 60 * 1000);
+	// setTimeout(() => {
+	// 	console.error(
+	// 		"Actor should've been destroyed by now. Automatically exiting.",
+	// 	);
+	//
+	// 	if (typeof Deno !== "undefined") Deno.exit(1);
+	// 	else process.exit(1);
+	// }, 60 * 1000);
 
 	let tickIndex = 0;
 	setInterval(() => {

frontend/apps/hub/src/domains/project/components/actors/actors-provider.tsx

@@ -12,6 +12,7 @@ import {
 	actorBuildsAtom,
 	createActorAtom,
 	type Logs,
+	type Metrics,
 	actorsQueryAtom,
 	actorsInternalFilterAtom,
 	type Actor,
@@ -30,6 +31,7 @@ import {
 	createActorEndpoint,
 	destroyActorMutationOptions,
 	actorLogsQueryOptions,
+	actorMetricsQueryOptions,
 	actorRegionsQueryOptions,
 	actorBuildsQueryOptions,
 } from "../../queries";
@@ -257,9 +259,63 @@ export function ActorsProvider({
 								};
 							};
 
+							const metrics = atom({
+								metrics: { cpu: null, memory: null } as Metrics,
+								status: "pending",
+							});
+							metrics.onMount = (set) => {
+								const metricsObserver = new QueryObserver(
+									queryClient,
+									actorMetricsQueryOptions({
+										projectNameId,
+										environmentNameId,
+										actorId: actor.id,
+									}, { refetchInterval: 5000 }),
+								);
+
+								type MetricsQuery = {
+									status: string;
+									data?: Awaited<
+										ReturnType<
+											Exclude<
+												ReturnType<
+													typeof actorMetricsQueryOptions
+												>["queryFn"],
+												undefined
+											>
+										>
+									>;
+								};
+
+								function updateMetrics(query: MetricsQuery) {
+									const data = query.data;
+									set((prev) => ({
+										...prev,
+										...data,
+										status: query.status,
+									}));
+								}
+
+								const subMetrics = metricsObserver.subscribe(
+									(query) => {
+										updateMetrics(query);
+									},
+								);
+
+								updateMetrics(
+									metricsObserver.getCurrentQuery().state,
+								);
+
+								return () => {
+									metricsObserver.destroy();
+									subMetrics();
+								};
+							};
+
 							return {
 								...actor,
 								logs,
+								metrics,
 								destroy,
 								status: getActorStatus(actor),
 							};

frontend/apps/hub/src/domains/project/queries/actors/query-options.ts

@@ -10,7 +10,7 @@ import {
 	keepPreviousData,
 	queryOptions,
 } from "@tanstack/react-query";
-import stripAnsi from 'strip-ansi';
+import stripAnsi from "strip-ansi";
 
 export const projectActorsQueryOptions = ({
 	projectNameId,
@@ -243,18 +243,142 @@ export const actorLogsQueryOptions = (
 					line: raw,
 					message: "",
 					properties: {},
-				} as const
+				} as const;
 			});
 
-
-			return {...response, logs };
+			return { ...response, logs };
 		},
 		meta: {
 			watch: mergeWatchStreams,
 		},
 	});
 };
 
+export const actorMetricsQueryOptions = (
+	{
+		projectNameId,
+		environmentNameId,
+		actorId,
+	}: {
+		projectNameId: string;
+		environmentNameId: string;
+		actorId: string;
+	},
+	opts: { refetchInterval?: number } = {},
+) => {
+	return queryOptions({
+		...opts,
+		queryKey: [
+			"project",
+			projectNameId,
+			"environment",
+			environmentNameId,
+			"actor",
+			actorId,
+			"metrics",
+		] as const,
+		queryFn: async ({
+			signal: abortSignal,
+			queryKey: [, project, , environment, , actorId],
+		}) => {
+			const pollOffset = 5_000;
+			const pollInterval = 15_000;
+
+			const now = Date.now();
+			const start = now - pollInterval * 2 - pollOffset; // Last minute + 2 data points
+			const end = now - pollOffset; // Metrics have a minimum a 5 second latency based on poll interval
+
+			const response = await rivetClient.actors.metrics.get(
+				actorId,
+				{
+					project,
+					environment,
+					start,
+					end,
+					interval: pollInterval,
+				},
+				{ abortSignal },
+			);
+
+			// Process the new response format
+			const metrics: Record<string, number | null> = {};
+			const rawData: Record<string, number[]> = {};
+
+			if (
+				response.metricNames &&
+				response.metricValues &&
+				response.metricAttributes &&
+				response.metricNames.length > 0
+			) {
+				response.metricNames.forEach((metricName, index) => {
+					const metricValues = response.metricValues[index];
+					const attributes = response.metricAttributes[index] || {};
+					
+					// Create the metric key based on the metric name and attributes
+					let metricKey = metricName;
+					
+					// Handle specific attribute mappings to match UI expectations
+					if (attributes.failure_type && attributes.scope) {
+						metricKey = `memory_failures_${attributes.failure_type}_${attributes.scope}`;
+					} else if (attributes.tcp_state) {
+						if (metricName.includes('tcp6')) {
+							metricKey = `network_tcp6_usage_${attributes.tcp_state}`;
+						} else {
+							metricKey = `network_tcp_usage_${attributes.tcp_state}`;
+						}
+					} else if (attributes.udp_state) {
+						if (metricName.includes('udp6')) {
+							metricKey = `network_udp6_usage_${attributes.udp_state}`;
+						} else {
+							metricKey = `network_udp_usage_${attributes.udp_state}`;
+						}
+					} else if (attributes.state) {
+						metricKey = `tasks_state_${attributes.state}`;
+					} else if (attributes.interface) {
+						// Handle network interface attributes
+						const baseMetric = metricName.replace(/^container_/, '');
+						metricKey = `${baseMetric}_${attributes.interface}`;
+					} else if (attributes.device) {
+						// Handle filesystem device attributes
+						const baseMetric = metricName.replace(/^container_/, '');
+						metricKey = `${baseMetric}_${attributes.device}`;
+					} else {
+						// Remove "container_" prefix to match UI expectations
+						metricKey = metricName.replace(/^container_/, '');
+					}
+					
+					// Store raw time series data for rate calculations
+					rawData[metricKey] = metricValues || [];
+					
+					if (metricValues && metricValues.length > 0) {
+						// Get the latest non-zero value (last value is often 0)
+						let value = null;
+						for (let i = metricValues.length - 1; i >= 0; i--) {
+							if (metricValues[i] !== 0) {
+								value = metricValues[i];
+								break;
+							}
+						}
+						// If all values are 0, use the last value anyway
+						if (value === null && metricValues.length > 0) {
+							value = metricValues[metricValues.length - 1];
+						}
+						metrics[metricKey] = value;
+					} else {
+						metrics[metricKey] = null;
+					}
+				});
+			}
+
+			return {
+				metrics,
+				rawData,
+				interval: pollInterval,
+			};
+		},
+	});
+};
+
 export const actorBuildsQueryOptions = ({
 	projectNameId,
 	environmentNameId,

frontend/packages/components/src/actors/actor-context.tsx

@@ -13,6 +13,7 @@ export enum ActorFeature {
 	State = "state",
 	Console = "console",
 	Runtime = "runtime",
+	Metrics = "metrics",
 	InspectReconnectNotification = "inspect_reconnect_notification",
 }
 
@@ -25,6 +26,7 @@ export type Actor = Omit<
 	lifecycle?: Rivet.actor.Lifecycle;
 	endpoint?: string;
 	logs: LogsAtom;
+	metrics: MetricsAtom;
 	network?: Rivet.actor.Network | null;
 	resources?: Rivet.actor.Resources | null;
 	runtime?: Rivet.actor.Runtime | null;
@@ -43,6 +45,8 @@ export type Logs = {
 	properties: Record<string, unknown>;
 }[];
 
+export type Metrics = Record<string, number | null>;
+
 export type Build = Rivet.actor.Build;
 export type DestroyActor = {
 	isDestroying: boolean;
@@ -55,6 +59,11 @@ export type LogsAtom = Atom<{
 	// query status
 	status: string;
 }>;
+export type MetricsAtom = Atom<{
+	metrics: Metrics;
+	// query status
+	status: string;
+}>;
 export type BuildAtom = Atom<Build>;
 export type DestroyActorAtom = Atom<DestroyActor>;
 
@@ -378,6 +387,7 @@ const commonActorFeatures = [
 	ActorFeature.Logs,
 	ActorFeature.Config,
 	ActorFeature.Runtime,
+	ActorFeature.Metrics,
 	ActorFeature.InspectReconnectNotification,
 ];