Skip to content

Commit c416210

Browse files
OskarStarkOskarStark
committed
feature #872 [Platform][LiteLLM] Add Bridge (welcoMattic)
This PR was merged into the main branch. Discussion ---------- [Platform][LiteLLM] Add Bridge | Q | A | ------------- | --- | Bug fix? | no | New feature? | yes <!-- please update src/**/CHANGELOG.md files --> | Docs? | no <!-- required for new features --> | Issues | <!-- prefix each issue number with "Fix #", no need to create an issue if none exist, explain below instead --> | License | MIT This PR adds support for [LiteLLM](https://docs.litellm.ai/docs/) as Bridge For now, it is a very simple support (only chat), see it as ready-to-evolve bridge. I've added a litellm service in Docker Compose file of examples to easily test it without having to install LiteLLM locally on your host. Commits ------- 0f4f0e0 Add LiteLLM Platform Bridge
2 parents d06d642 + 0f4f0e0 commit c416210

File tree

10 files changed

+302
-0
lines changed

10 files changed

+302
-0
lines changed

‎examples/.env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,9 @@ MEILISEARCH_API_KEY=changeMe
112112
# For using LMStudio
113113
LMSTUDIO_HOST_URL=http://127.0.0.1:1234
114114

115+
# For using LiteLLM
116+
LITELLM_HOST_URL=http://127.0.0.1:4000
117+
115118
# Qdrant (store)
116119
QDRANT_HOST=http://127.0.0.1:6333
117120
QDRANT_SERVICE_API_KEY=changeMe

‎examples/compose.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,16 @@ services:
188188
- '8080:8080'
189189
- '50051:50051'
190190

191+
litellm:
192+
image: ghcr.io/berriai/litellm:v1.79.1-stable
193+
ports:
194+
- "4000:4000"
195+
volumes:
196+
- ./litellm/config.yaml:/app/config.yaml
197+
env_file:
198+
- .env
199+
command: [ "--config", "/app/config.yaml", "--port", "4000", "--num_workers", "8" ]
200+
191201
volumes:
192202
typesense_data:
193203
etcd_vlm:

‎examples/litellm/chat.php‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Platform\Bridge\LiteLlm\PlatformFactory;
13+
use Symfony\AI\Platform\Message\Message;
14+
use Symfony\AI\Platform\Message\MessageBag;
15+
16+
require_once dirname(__DIR__).'/bootstrap.php';
17+
18+
$platform = PlatformFactory::create(env('LITELLM_HOST_URL'), http_client());
19+
20+
$messages = new MessageBag(
21+
Message::forSystem('You are a pirate and you write funny.'),
22+
Message::ofUser('What is the Symfony framework?'),
23+
);
24+
$result = $platform->invoke('mistral-small-latest', $messages, [
25+
'max_tokens' => 500, // specific options just for this call
26+
]);
27+
28+
echo $result->asText().\PHP_EOL;

‎examples/litellm/config.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
model_list:
2+
- model_name: mistral-small-latest
3+
litellm_params:
4+
model: mistral/mistral-small-latest
5+
api_key: "os.environ/MISTRAL_API_KEY"

‎src/platform/composer.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
"gemini",
1717
"huggingface",
1818
"inference",
19+
"litellm",
1920
"llama",
2021
"lmstudio",
2122
"meta",

‎src/platform/src/Bridge/LiteLlm/ModelCatalog.php‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\AI\Platform\Bridge\LiteLlm;
13+
14+
use Symfony\AI\Platform\ModelCatalog\FallbackModelCatalog;
15+
16+
/**
17+
* @author Mathieu Santostefano <[email protected]>
18+
*/
19+
final class ModelCatalog extends FallbackModelCatalog
20+
{
21+
// LiteLLM can use any model that is loaded locally
22+
// Models are dynamically available based on what's configured in LiteLLM
23+
}

‎src/platform/src/Bridge/LiteLlm/ModelClient.php‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\AI\Platform\Bridge\LiteLlm;
13+
14+
use Symfony\AI\Platform\Model;
15+
use Symfony\AI\Platform\ModelClientInterface;
16+
use Symfony\AI\Platform\Result\RawHttpResult;
17+
use Symfony\Component\HttpClient\EventSourceHttpClient;
18+
use Symfony\Contracts\HttpClient\HttpClientInterface;
19+
20+
/**
21+
* @author Mathieu Santostefano <[email protected]>
22+
*/
23+
final class ModelClient implements ModelClientInterface
24+
{
25+
private readonly EventSourceHttpClient $httpClient;
26+
27+
public function __construct(
28+
HttpClientInterface $httpClient,
29+
private readonly string $hostUrl,
30+
) {
31+
$this->httpClient = $httpClient instanceof EventSourceHttpClient ? $httpClient : new EventSourceHttpClient($httpClient);
32+
}
33+
34+
public function supports(Model $model): bool
35+
{
36+
return true;
37+
}
38+
39+
public function request(Model $model, array|string $payload, array $options = []): RawHttpResult
40+
{
41+
return new RawHttpResult($this->httpClient->request('POST', \sprintf('%s/v1/chat/completions', $this->hostUrl), [
42+
'json' => array_merge($options, $payload),
43+
]));
44+
}
45+
}

‎src/platform/src/Bridge/LiteLlm/PlatformFactory.php‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\AI\Platform\Bridge\LiteLlm;
13+
14+
use Psr\EventDispatcher\EventDispatcherInterface;
15+
use Symfony\AI\Platform\Contract;
16+
use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface;
17+
use Symfony\AI\Platform\Platform;
18+
use Symfony\Component\HttpClient\EventSourceHttpClient;
19+
use Symfony\Contracts\HttpClient\HttpClientInterface;
20+
21+
/**
22+
* @author Mathieu Santostefano <[email protected]>
23+
*/
24+
class PlatformFactory
25+
{
26+
public static function create(
27+
string $hostUrl = 'http://localhost:4000',
28+
?HttpClientInterface $httpClient = null,
29+
ModelCatalogInterface $modelCatalog = new ModelCatalog(),
30+
?Contract $contract = null,
31+
?EventDispatcherInterface $eventDispatcher = null,
32+
): Platform {
33+
$httpClient = $httpClient instanceof EventSourceHttpClient ? $httpClient : new EventSourceHttpClient($httpClient);
34+
35+
return new Platform(
36+
[
37+
new ModelClient($httpClient, $hostUrl),
38+
],
39+
[
40+
new ResultConverter(),
41+
],
42+
$modelCatalog,
43+
$contract,
44+
$eventDispatcher,
45+
);
46+
}
47+
}

‎src/platform/src/Bridge/LiteLlm/ResultConverter.php‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\AI\Platform\Bridge\LiteLlm;
13+
14+
use Symfony\AI\Platform\Bridge\OpenAi\Gpt\ResultConverter as OpenAiResponseConverter;
15+
use Symfony\AI\Platform\Model;
16+
use Symfony\AI\Platform\Result\RawResultInterface;
17+
use Symfony\AI\Platform\Result\ResultInterface;
18+
use Symfony\AI\Platform\ResultConverterInterface;
19+
20+
/**
21+
* @author Mathieu Santostefano <[email protected]>
22+
*/
23+
final class ResultConverter implements ResultConverterInterface
24+
{
25+
public function __construct(
26+
private readonly OpenAiResponseConverter $gptResponseConverter = new OpenAiResponseConverter(),
27+
) {
28+
}
29+
30+
public function supports(Model $model): bool
31+
{
32+
return true;
33+
}
34+
35+
public function convert(RawResultInterface $result, array $options = []): ResultInterface
36+
{
37+
return $this->gptResponseConverter->convert($result, $options);
38+
}
39+
}

‎src/platform/tests/Bridge/LiteLlm/ModelClientTest.php‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\AI\Platform\Tests\Bridge\LiteLlm;
13+
14+
use PHPUnit\Framework\TestCase;
15+
use Symfony\AI\Platform\Bridge\LiteLlm\ModelClient;
16+
use Symfony\AI\Platform\Model;
17+
use Symfony\Component\HttpClient\EventSourceHttpClient;
18+
use Symfony\Component\HttpClient\MockHttpClient;
19+
use Symfony\Component\HttpClient\Response\MockResponse;
20+
21+
class ModelClientTest extends TestCase
22+
{
23+
public function testItIsSupportingTheCorrectModel()
24+
{
25+
$client = new ModelClient(new MockHttpClient(), 'http://localhost:4000');
26+
27+
$this->assertTrue($client->supports(new Model('test-model')));
28+
}
29+
30+
public function testItIsExecutingTheCorrectRequest()
31+
{
32+
$resultCallback = static function (string $method, string $url, array $options): MockResponse {
33+
self::assertSame('POST', $method);
34+
self::assertSame('http://localhost:4000/v1/chat/completions', $url);
35+
self::assertSame(
36+
'{"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}',
37+
$options['body']
38+
);
39+
40+
return new MockResponse();
41+
};
42+
43+
$httpClient = new MockHttpClient([$resultCallback]);
44+
$client = new ModelClient($httpClient, 'http://localhost:4000');
45+
46+
$payload = [
47+
'model' => 'test-model',
48+
'messages' => [
49+
['role' => 'user', 'content' => 'Hello, world!'],
50+
],
51+
];
52+
53+
$client->request(new Model('test-model'), $payload);
54+
}
55+
56+
public function testItMergesOptionsWithPayload()
57+
{
58+
$resultCallback = static function (string $method, string $url, array $options): MockResponse {
59+
self::assertSame('POST', $method);
60+
self::assertSame('http://localhost:4000/v1/chat/completions', $url);
61+
self::assertSame(
62+
'{"temperature":0.7,"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}',
63+
$options['body']
64+
);
65+
66+
return new MockResponse();
67+
};
68+
69+
$httpClient = new MockHttpClient([$resultCallback]);
70+
$client = new ModelClient($httpClient, 'http://localhost:4000');
71+
72+
$payload = [
73+
'model' => 'test-model',
74+
'messages' => [
75+
['role' => 'user', 'content' => 'Hello, world!'],
76+
],
77+
];
78+
79+
$client->request(new Model('test-model'), $payload, ['temperature' => 0.7]);
80+
}
81+
82+
public function testItUsesEventSourceHttpClient()
83+
{
84+
$httpClient = new MockHttpClient();
85+
$client = new ModelClient($httpClient, 'http://localhost:4000');
86+
87+
$reflection = new \ReflectionProperty($client, 'httpClient');
88+
89+
$this->assertInstanceOf(EventSourceHttpClient::class, $reflection->getValue($client));
90+
}
91+
92+
public function testItKeepsExistingEventSourceHttpClient()
93+
{
94+
$eventSourceHttpClient = new EventSourceHttpClient(new MockHttpClient());
95+
$client = new ModelClient($eventSourceHttpClient, 'http://localhost:4000');
96+
97+
$reflection = new \ReflectionProperty($client, 'httpClient');
98+
99+
$this->assertSame($eventSourceHttpClient, $reflection->getValue($client));
100+
}
101+
}

0 commit comments

Comments
 (0)