From 4dbd8942581b432cff74aaae1d398ac3002b7581 Mon Sep 17 00:00:00 2001 From: Anas Date: Mon, 4 Aug 2025 00:03:29 +0300 Subject: [PATCH] feat: Add LM Studio provider support for local model hosting - Add LM Studio provider configuration in config.example.json - Include setup script (setup-lmstudio.sh) for easy configuration - Add comprehensive documentation (README-LMSTUDIO.md) - Provide integration testing script (test-lmstudio-integration.sh) - Update main README to mention LM Studio support - Add sample configuration file (config.lmstudio.json) LM Studio is a popular local model hosting platform that provides OpenAI-compatible endpoints. This integration allows users to route Claude Code requests to their local LM Studio models alongside existing providers like Ollama. --- README-LMSTUDIO.md | 219 +++++++++++++++++++++++++++++++++++ README.md | 8 +- config.example.json | 10 +- config.lmstudio.json | 20 ++++ setup-lmstudio.sh | 25 ++++ test-lmstudio-integration.sh | 94 +++++++++++++++ 6 files changed, 373 insertions(+), 3 deletions(-) create mode 100644 README-LMSTUDIO.md create mode 100644 config.lmstudio.json create mode 100755 setup-lmstudio.sh create mode 100755 test-lmstudio-integration.sh diff --git a/README-LMSTUDIO.md b/README-LMSTUDIO.md new file mode 100644 index 00000000..51ac9e40 --- /dev/null +++ b/README-LMSTUDIO.md @@ -0,0 +1,219 @@ +# LM Studio Integration with Claude Code Router + +This guide explains how to integrate Claude Code Router (CCR) with LM Studio to use local models with Claude Code. + +## Prerequisites + +1. [LM Studio](https://lmstudio.ai/) installed and running +2. [Claude Code](https://docs.anthropic.com/en/docs/claude-code/quickstart) installed +3. Claude Code Router installed: `npm install -g @musistudio/claude-code-router` + +## Quick Setup + +1. **Start LM Studio**: + - Open LM Studio + - Load a model (e.g., `google/gemma-3n-e4b`) + - Start the local server (typically on `http://localhost:1234`) + +2. **Configure CCR for LM Studio**: + ```bash + ./setup-lmstudio.sh + ``` + +3. **Start CCR**: + ```bash + ccr start + ``` + +4. **Use Claude Code with LM Studio**: + ```bash + ccr code "What is the capital of France?" + ``` + +## Manual Configuration + +If you prefer to configure manually, create `~/.claude-code-router/config.json`: + +```json +{ + "Providers": [ + { + "name": "lmstudio", + "api_base_url": "http://localhost:1234/v1/chat/completions", + "api_key": "lm-studio", + "models": ["your-model-name"] + } + ], + "Router": { + "default": "lmstudio,your-model-name", + "background": "lmstudio,your-model-name", + "think": "lmstudio,your-model-name", + "longContext": "lmstudio,your-model-name", + "longContextThreshold": 60000, + "webSearch": "lmstudio,your-model-name" + }, + "API_TIMEOUT_MS": 600000, + "LOG": true +} +``` + +## Configuration Options + +### Model Configuration +Update the `models` array with your loaded LM Studio model: +```json +"models": ["microsoft/DialoGPT-medium", "meta-llama/Llama-2-7b-chat-hf"] +``` + +### Custom API Endpoint +If LM Studio runs on a different port: +```json +"api_base_url": "http://localhost:8080/v1/chat/completions" +``` + +### Multiple Models Setup +Configure different models for different tasks: +```json +{ + "Providers": [ + { + "name": "lmstudio", + "api_base_url": "http://localhost:1234/v1/chat/completions", + "api_key": "lm-studio", + "models": ["google/gemma-3n-e4b", "microsoft/DialoGPT-medium"] + } + ], + "Router": { + "default": "lmstudio,google/gemma-3n-e4b", + "background": "lmstudio,microsoft/DialoGPT-medium", + "think": "lmstudio,google/gemma-3n-e4b" + } +} +``` + +## Advanced Configuration + +### Adding Transformers +While LM Studio doesn't typically need transformers (it's OpenAI-compatible), you can add them for custom behavior: + +```json +{ + "name": "lmstudio", + "api_base_url": "http://localhost:1234/v1/chat/completions", + "api_key": "lm-studio", + "models": ["google/gemma-3n-e4b"], + "transformer": { + "use": [ + [ + "maxtoken", + { + "max_tokens": 4096 + } + ] + ] + } +} +``` + +### Custom Router +Create a custom router for LM Studio specific logic: + +```javascript +// ~/.claude-code-router/custom-lmstudio-router.js +module.exports = async function router(req, config) { + const userMessage = req.body.messages.find((m) => m.role === "user")?.content; + + // Use a specific model for code-related queries + if (userMessage && userMessage.toLowerCase().includes("code")) { + return "lmstudio,google/gemma-3n-e4b"; + } + + // Default to configured router + return null; +}; +``` + +Then in your config: +```json +{ + "CUSTOM_ROUTER_PATH": "$HOME/.claude-code-router/custom-lmstudio-router.js" +} +``` + +## Testing Your Setup + +1. **Test basic functionality**: + ```bash + ccr code "Hello, how are you?" + ``` + +2. **Test with longer context**: + ```bash + ccr code "Explain the concept of machine learning in detail" + ``` + +3. **Check routing**: + ```bash + ccr status + ``` + +## Troubleshooting + +### Common Issues + +1. **Connection refused**: Make sure LM Studio server is running +2. **Model not found**: Ensure the model name in config matches the loaded model in LM Studio +3. **Timeout errors**: Increase `API_TIMEOUT_MS` in config for slower local models + +### Debug Mode +Enable logging for troubleshooting: +```json +{ + "LOG": true +} +``` + +View logs: +```bash +tail -f ~/.claude-code-router.log +``` + +### Testing LM Studio API Directly +Test the LM Studio API directly with curl (same as your original command): + +```bash +curl http://localhost:1234/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "google/gemma-3n-e4b", + "messages": [ + { "role": "system", "content": "Always answer in rhymes. Today is Thursday" }, + { "role": "user", "content": "What day is it today?" } + ], + "temperature": 0.7, + "max_tokens": -1, + "stream": false +}' +``` + +## Benefits of Using CCR with LM Studio + +1. **Unified Interface**: Use Claude Code with local models +2. **Routing Logic**: Automatically route different types of requests to appropriate models +3. **Easy Model Switching**: Switch between local and cloud models seamlessly +4. **Cost Effective**: Use local models for development and testing +5. **Privacy**: Keep sensitive code on your local machine + +## Example Workflows + +### Development Workflow +```bash +# Use local model for quick iterations +ccr code "Review this Python function for bugs" + +# Switch to cloud model for complex tasks +/model openrouter,anthropic/claude-3.5-sonnet +``` + +### Mixed Model Setup +Configure different models for different scenarios in your config.json and let CCR automatically route requests based on context, token count, and task type. \ No newline at end of file diff --git a/README.md b/README.md index 8a16fc3f..33561caa 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ ## โœจ Features - **Model Routing**: Route requests to different models based on your needs (e.g., background tasks, thinking, long context). -- **Multi-Provider Support**: Supports various model providers like OpenRouter, DeepSeek, Ollama, Gemini, Volcengine, and SiliconFlow. +- **Multi-Provider Support**: Supports various model providers like OpenRouter, DeepSeek, Ollama, Gemini, Volcengine, SiliconFlow, and LM Studio. - **Request/Response Transformation**: Customize requests and responses for different providers using transformers. - **Dynamic Model Switching**: Switch models on-the-fly within Claude Code using the `/model` command. - **GitHub Actions Integration**: Trigger Claude Code tasks in your GitHub workflows. @@ -151,6 +151,12 @@ Here is a comprehensive example: "claude-opus-4-20250514", "gemini-2.5-pro" ] + }, + { + "name": "lmstudio", + "api_base_url": "http://localhost:1234/v1/chat/completions", + "api_key": "lm-studio", + "models": ["google/gemma-3n-e4b"] } ], "Router": { diff --git a/config.example.json b/config.example.json index 2c1df1f2..1c5c7ea4 100644 --- a/config.example.json +++ b/config.example.json @@ -102,11 +102,17 @@ "enhancetool" ] } + }, + { + "name": "lmstudio", + "api_base_url": "http://localhost:1234/v1/chat/completions", + "api_key": "lm-studio", + "models": ["google/gemma-3n-e4b"] } ], "Router": { - "default": "deepseek,deepseek-chat", - "background": "ollama,qwen2.5-coder:latest", + "default": "lmstudio,google/gemma-3n-e4b", + "background": "lmstudio,google/gemma-3n-e4b", "think": "deepseek,deepseek-reasoner", "longContext": "openrouter,google/gemini-2.5-pro-preview", "longContextThreshold": 60000, diff --git a/config.lmstudio.json b/config.lmstudio.json new file mode 100644 index 00000000..b9a26ed9 --- /dev/null +++ b/config.lmstudio.json @@ -0,0 +1,20 @@ +{ + "Providers": [ + { + "name": "lmstudio", + "api_base_url": "http://localhost:1234/v1/chat/completions", + "api_key": "lm-studio", + "models": ["google/gemma-3n-e4b"] + } + ], + "Router": { + "default": "lmstudio,google/gemma-3n-e4b", + "background": "lmstudio,google/gemma-3n-e4b", + "think": "lmstudio,google/gemma-3n-e4b", + "longContext": "lmstudio,google/gemma-3n-e4b", + "longContextThreshold": 60000, + "webSearch": "lmstudio,google/gemma-3n-e4b" + }, + "API_TIMEOUT_MS": 600000, + "LOG": true +} \ No newline at end of file diff --git a/setup-lmstudio.sh b/setup-lmstudio.sh new file mode 100755 index 00000000..5be2e397 --- /dev/null +++ b/setup-lmstudio.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Setup script for integrating Claude Code Router with LM Studio +echo "Setting up Claude Code Router for LM Studio..." + +# Create .claude-code-router directory if it doesn't exist +mkdir -p "$HOME/.claude-code-router" + +# Copy the LM Studio configuration +echo "Copying LM Studio configuration..." +cp config.lmstudio.json "$HOME/.claude-code-router/config.json" + +echo "Configuration copied to $HOME/.claude-code-router/config.json" +echo "" +echo "To use CCR with LM Studio:" +echo "1. Make sure LM Studio is running on http://localhost:1234" +echo "2. Load the model 'google/gemma-3n-e4b' in LM Studio" +echo "3. Run: ccr start" +echo "4. Run: ccr code 'your prompt here'" +echo "" +echo "You can also edit the configuration at:" +echo "$HOME/.claude-code-router/config.json" +echo "" +echo "To change the model, update the 'models' array in the lmstudio provider" +echo "and update the Router section accordingly." \ No newline at end of file diff --git a/test-lmstudio-integration.sh b/test-lmstudio-integration.sh new file mode 100755 index 00000000..0c9b4e87 --- /dev/null +++ b/test-lmstudio-integration.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +echo "๐Ÿงช Testing LM Studio Integration with Claude Code Router" +echo "==================================================" + +# Check if LM Studio is running +echo "1. Testing LM Studio API directly..." +if curl -s -f http://localhost:1234/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "google/gemma-3n-e4b", + "messages": [ + { "role": "user", "content": "Hello, respond with just: LM Studio is working!" } + ], + "temperature": 0.7, + "max_tokens": 50, + "stream": false + }' > /tmp/lmstudio_test.json 2>/dev/null; then + echo "โœ… LM Studio API is responding" + echo "Response: $(cat /tmp/lmstudio_test.json | grep -o '"content":"[^"]*"' | head -1)" +else + echo "โŒ LM Studio API is not responding on http://localhost:1234" + echo " Make sure LM Studio is running with the model loaded" + exit 1 +fi + +echo "" +echo "2. Checking CCR configuration..." +if [ -f "$HOME/.claude-code-router/config.json" ]; then + echo "โœ… CCR config exists at $HOME/.claude-code-router/config.json" + if grep -q "lmstudio" "$HOME/.claude-code-router/config.json"; then + echo "โœ… LM Studio provider found in config" + else + echo "โš ๏ธ LM Studio provider not found in config" + echo " Run ./setup-lmstudio.sh to configure" + fi +else + echo "โŒ CCR config not found" + echo " Run ./setup-lmstudio.sh to create configuration" + exit 1 +fi + +echo "" +echo "3. Testing CCR status..." +if ccr status 2>/dev/null | grep -q "running"; then + echo "โœ… CCR is running" +elif ccr status 2>/dev/null | grep -q "stopped"; then + echo "๐Ÿ”„ Starting CCR..." + ccr start + sleep 3 + if ccr status 2>/dev/null | grep -q "running"; then + echo "โœ… CCR started successfully" + else + echo "โŒ Failed to start CCR" + exit 1 + fi +else + echo "โŒ CCR command not found or error occurred" + echo " Make sure @musistudio/claude-code-router is installed globally" + exit 1 +fi + +echo "" +echo "4. Testing CCR with LM Studio integration..." +echo " Sending test prompt to CCR..." + +# Test CCR integration +TEST_RESPONSE=$(ccr code "Just respond with: CCR + LM Studio integration working!" 2>&1) + +if echo "$TEST_RESPONSE" | grep -q "working"; then + echo "โœ… CCR + LM Studio integration is working!" + echo "Response preview: $(echo "$TEST_RESPONSE" | head -3)" +else + echo "โŒ CCR + LM Studio integration test failed" + echo "Response: $TEST_RESPONSE" + echo "" + echo "Troubleshooting steps:" + echo "1. Verify LM Studio has the correct model loaded" + echo "2. Check CCR logs: tail -f ~/.claude-code-router.log" + echo "3. Verify configuration: cat ~/.claude-code-router/config.json" + exit 1 +fi + +echo "" +echo "๐ŸŽ‰ All tests passed! LM Studio integration is working correctly." +echo "" +echo "You can now use Claude Code with LM Studio:" +echo " ccr code 'Your prompt here'" +echo "" +echo "To switch models dynamically:" +echo " /model lmstudio,your-model-name" + +# Clean up +rm -f /tmp/lmstudio_test.json \ No newline at end of file