Merge branch 'main' into ning/release-benchmarking

Signed-off-by: Ning <[email protected]>

Author: nwangfw

.github/workflows/release-build.yaml

@@ -140,6 +140,10 @@ jobs:
       id-token: write
     name: publish
     steps:
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
       - name: Check out source repository
         uses: actions/checkout@v4
       - name: Set up Python environment ${{ matrix.python-version }}

config/features/vllm-remote-tokenizer/README.md

@@ -0,0 +1,68 @@
+# vLLM Remote Tokenizer Feature
+
+This feature enables model-aware remote tokenizer support for vLLM inference engines in AIBrix gateway.
+
+## Quick Start
+
+Enable vLLM remote tokenizer with one command:
+
+```bash
+kubectl apply -k config/features/vllm-remote-tokenizer/
+```
+
+## Configuration
+
+The following environment variables are configured:
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| AIBRIX_ENABLE_VLLM_REMOTE_TOKENIZER | false | Enable remote tokenizer feature |
+| AIBRIX_VLLM_TOKENIZER_ENDPOINT_TEMPLATE | http://%s:8000 | URL template for vLLM endpoints |
+| AIBRIX_TOKENIZER_HEALTH_CHECK_PERIOD | 30s | Health check interval |
+| AIBRIX_TOKENIZER_TTL | 5m | Tokenizer cache TTL |
+| AIBRIX_MAX_TOKENIZERS_PER_POOL | 100 | Maximum tokenizers per pool |
+| AIBRIX_TOKENIZER_REQUEST_TIMEOUT | 10s | Request timeout |
+
+## Customization
+
+To use custom values, copy this directory and modify `gateway-plugins-env-patch.yaml`:
+
+```bash
+cp -r config/features/vllm-remote-tokenizer/ config/features/my-vllm-config/
+# Edit config/features/my-vllm-config/gateway-plugins-env-patch.yaml
+kubectl apply -k config/features/my-vllm-config/
+```
+
+## Enable the Feature
+
+To enable vLLM remote tokenizer after installation:
+
+```bash
+kubectl set env deployment/gateway-plugins -n aibrix-system AIBRIX_ENABLE_VLLM_REMOTE_TOKENIZER=true
+```
+
+Or use a custom Kustomization overlay with the environment variable set to `true`.
+
+## Disable
+
+To disable, set the environment variable to false:
+
+```bash
+kubectl set env deployment/gateway-plugins -n aibrix-system AIBRIX_ENABLE_VLLM_REMOTE_TOKENIZER=false
+```
+
+## Verification
+
+Check if enabled:
+
+```bash
+kubectl get deployment gateway-plugins -n aibrix-system -o json | \
+  jq '.spec.template.spec.containers[0].env[] | select(.name | startswith("AIBRIX_ENABLE_VLLM"))'
+```
+
+Check metrics:
+
+```bash
+kubectl port-forward -n aibrix-system svc/gateway-plugins 8080:8080
+curl http://localhost:8080/metrics | grep aibrix_tokenizer_pool
+```

config/features/vllm-remote-tokenizer/gateway-plugins-env-patch.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gateway-plugins
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: gateway-plugin
+        env:
+        - name: AIBRIX_ENABLE_VLLM_REMOTE_TOKENIZER
+          value: "false"
+        - name: AIBRIX_VLLM_TOKENIZER_ENDPOINT_TEMPLATE
+          value: "http://%s:8000"
+        - name: AIBRIX_TOKENIZER_HEALTH_CHECK_PERIOD
+          value: "30s"
+        - name: AIBRIX_TOKENIZER_TTL
+          value: "5m"
+        - name: AIBRIX_MAX_TOKENIZERS_PER_POOL
+          value: "100"
+        - name: AIBRIX_TOKENIZER_REQUEST_TIMEOUT
+          value: "10s"

config/features/vllm-remote-tokenizer/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: aibrix-system
+
+# This overlay enables vLLM remote tokenizer support
+# Apply with: kubectl apply -k config/features/vllm-remote-tokenizer/
+
+resources:
+- ../../gateway/gateway-plugin
+
+patches:
+- path: gateway-plugins-env-patch.yaml
+  target:
+    kind: Deployment
+    name: gateway-plugins

config/overlays/release/kustomization.yaml

@@ -20,10 +20,10 @@ images:
 - name: redis
   newTag: latest
 - name: aibrix/gateway-plugins
-  newTag: v0.3.0
+  newTag: v0.4.0.rc.1
 - name: aibrix/metadata-service
-  newTag: v0.3.0
+  newTag: v0.4.0.rc.1
 - name: aibrix/controller-manager
-  newTag: v0.3.0
+  newTag: v0.4.0.rc.1
 - name: aibrix/runtime
-  newTag: v0.3.0
+  newTag: v0.4.0.rc.1

config/overlays/vke/default/kustomization.yaml

@@ -20,13 +20,13 @@ images:
   newTag: latest
 - name: aibrix/gateway-plugins
   newName: aibrix-cn-beijing.cr.volces.com/aibrix/gateway-plugins
-  newTag: v0.3.0
+  newTag: v0.4.0.rc.1
 - name: aibrix/metadata-service
   newName: aibrix-cn-beijing.cr.volces.com/aibrix/metadata-service
-  newTag: v0.3.0
+  newTag: v0.4.0.rc.1
 - name: aibrix/controller-manager
   newName: aibrix-cn-beijing.cr.volces.com/aibrix/controller-manager
-  newTag: v0.3.0
+  newTag: v0.4.0.rc.1
 - name: aibrix/runtime
   newName: aibrix-cn-beijing.cr.volces.com/aibrix/runtime
-  newTag: v0.3.0
+  newTag: v0.4.0.rc.1

config/standalone/autoscaler-controller/kustomization.yaml

@@ -20,7 +20,7 @@ namePrefix: aibrix-autoscaling-
 images:
   - name: controller
     newName: aibrix/controller-manager
-    newTag: nightly
+    newTag: v0.4.0.rc.1
 
 patches:
   - path: patch.yaml

config/standalone/distributed-inference-controller/kustomization.yaml

@@ -21,7 +21,7 @@ namePrefix: aibrix-orchestration-
 images:
   - name: controller
     newName: aibrix/controller-manager
-    newTag: v0.3.0
+    newTag: v0.4.0.rc.1
   - name: quay.io/kuberay/operator
     newName: aibrix/kuberay-operator
     newTag: v1.2.1-patch-20250726

config/standalone/kv-cache-controller/kustomization.yaml

@@ -20,7 +20,7 @@ namePrefix: aibrix-kv-cache-
 images:
   - name: controller
     newName: aibrix/controller-manager
-    newTag: nightly
+    newTag: v0.4.0.rc.1
 
 patches:
   - path: patch.yaml

config/standalone/model-adapter-controller/kustomization.yaml

@@ -23,7 +23,7 @@ namePrefix: aibrix-lora-
 images:
   - name: controller
     newName: aibrix/controller-manager
-    newTag: nightly
+    newTag: v0.4.0.rc.1
 
 patches:
   - path: patch.yaml