feature #811 [Platform] Support multimodal embeddings with Voyage AI (paulinevos)

This PR was squashed before being merged into the main branch.

Discussion
----------

[Platform] Support multimodal embeddings with Voyage AI

| Q             | A
| ------------- | ---
| Bug fix?      | no
| New feature?  | yes <!-- please update src/**/CHANGELOG.md files -->
| Docs?         | yes <!-- required for new features -->
| License       | MIT

This PR adds support for multimodal embeddings using Voyage.

Commits
-------

bd91b18 [Platform] Support multimodal embeddings with Voyage AI

Author: chr-hertel

docs/components/platform/voyage.rst

@@ -0,0 +1,65 @@
+Voyage AI
+=========
+
+Voyage AI offers a number of models for embedding text, contextualized chunks, interleaved multimodal data, and reranking.
+The bundle currently supports text embedding and multimodal embedding.
+
+For comprehensive information about Voyage AI, see the `Voyage AI API reference`_
+
+Setup
+-----
+
+Authentication
+~~~~~~~~~~~~~~
+
+Voyage AI requires an API key, which you can set up in `Voyage AI dashboard`_.
+
+Usage
+-----
+
+Basic text embedding usage example::
+
+    use Symfony\AI\Platform\Bridge\Voyage\PlatformFactory;
+
+    $platform = PlatformFactory::create($_ENV['VOYAGE_API_KEY'], $httpClient);
+
+    $result = $platform->invoke('voyage-3', <<<TEXT
+        Once upon a time, there was a country called Japan. It was a beautiful country with a lot of mountains and
+        rivers. The people of Japan were very kind and hardworking. They loved their country very much and took care of
+        it. The country was very peaceful and prosperous. The people lived happily ever after.
+        TEXT);
+
+    echo $result->getContent();
+
+Voyage AI supports text, base64 image data, and image URLs in its multimodal embedding model. It also allows for
+multiple data types per vector embedding. To do this, wrap the data in a `Collection` as shown in the example below.
+
+Basic multimodal embedding usage example::
+
+    use Symfony\AI\Platform\Bridge\Voyage\PlatformFactory;
+    use Symfony\AI\Platform\Message\Content\Collection;
+    use Symfony\AI\Platform\Message\Content\ImageUrl;
+    use Symfony\AI\Platform\Message\Content\Text;
+
+    $platform = PlatformFactory::create($_ENV['VOYAGE_API_KEY'], $httpClient);
+
+    $result = $platform->invoke(
+        'voyage-multimodal-3',
+        new ImageUrl('https://example.com/image1.jpg'),
+        new Collection(new Text('Hello, world!'), new ImageUrl('https://example.com/image2.jpg')
+    );
+
+    echo $result->getContent();
+
+
+Examples
+--------
+
+See the ``examples/voyage/`` directory for complete working examples:
+
+* ``text-embeddings.php`` - Basic text embedding example
+* ``multiple-text-embeddings.php`` - Embedding multiple text values
+* ``multimodal-embeddings.php`` - Embedding multimodal data (single and multiple values)
+
+.. _Voyage AI API reference: https://docs.voyageai.com/reference/embeddings-api
+.. _Voyage AI dashboard: https://dashboard.voyageai.com/organization/api-keys

examples/voyage/multimodal-embeddings.php

@@ -0,0 +1,37 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Platform\Bridge\Voyage\PlatformFactory;
+use Symfony\AI\Platform\Message\Content\Collection;
+use Symfony\AI\Platform\Message\Content\Image;
+use Symfony\AI\Platform\Message\Content\Text;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$platform = PlatformFactory::create(env('VOYAGE_API_KEY'), http_client());
+
+$image = Image::fromFile(dirname(__DIR__, 2).'/fixtures/image.jpg');
+
+// Single value
+$result1 = $platform->invoke('voyage-multimodal-3',
+    new Text('Lorem ipsum dolor sit amet, consectetur adipiscing elit.'),
+);
+
+// Multiple values
+$result2 = $platform->invoke('voyage-multimodal-3', [
+    new Collection(new Text('Photo of a sunrise'), $image),
+    new Collection(new Text('Photo of a sunset'), $image),
+]);
+
+echo 'Dimensions for text: '.$result1->asVectors()[0]->getDimensions().\PHP_EOL;
+
+echo 'Dimensions for sunrise image and description: '.$result2->asVectors()[0]->getDimensions().\PHP_EOL;
+echo 'Dimensions for sunset image and description: '.$result2->asVectors()[1]->getDimensions().\PHP_EOL;

examples/voyage/multiple-embeddings.php renamed to examples/voyage/multiple-text-embeddings.php

@@ -66,3 +66,4 @@ CHANGELOG
  * Add tool calling support for Ollama platform
  * Allow beta feature flags to be passed into Anthropic model options
  * Add Ollama streaming output support
+ * Add multimodal embedding support for Voyage AI

examples/voyage/embeddings.php renamed to examples/voyage/text-embeddings.php

@@ -0,0 +1,54 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\Voyage\Contract\Multimodal;
+
+use Symfony\AI\Platform\Bridge\Voyage\Voyage;
+use Symfony\AI\Platform\Capability;
+use Symfony\AI\Platform\Contract;
+use Symfony\AI\Platform\Message\Content\Collection;
+use Symfony\AI\Platform\Model;
+use Symfony\Component\Serializer\Exception\ExceptionInterface;
+use Symfony\Component\Serializer\Normalizer\NormalizerAwareInterface;
+use Symfony\Component\Serializer\Normalizer\NormalizerAwareTrait;
+
+final class CollectionNormalizer extends Contract\Normalizer\ModelContractNormalizer implements NormalizerAwareInterface
+{
+    use NormalizerAwareTrait;
+
+    public const KEY_CONTENT = 'content';
+
+    /**
+     * @param Collection $data
+     *
+     * @throws ExceptionInterface
+     */
+    public function normalize(mixed $data, ?string $format = null, array $context = []): array
+    {
+        $content = [];
+        foreach ($data->getContent() as $item) {
+            $normalized = $this->normalizer->normalize($item, $format, $context);
+            $content = array_merge($content, array_pop($normalized)[self::KEY_CONTENT]);
+        }
+
+        return [['content' => $content]];
+    }
+
+    protected function supportedDataClass(): string
+    {
+        return Collection::class;
+    }
+
+    protected function supportsModel(Model $model): bool
+    {
+        return $model instanceof Voyage && $model->supports(Capability::INPUT_MULTIMODAL);
+    }
+}

src/platform/CHANGELOG.md

@@ -0,0 +1,50 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\Voyage\Contract\Multimodal;
+
+use Symfony\AI\Platform\Bridge\Voyage\Voyage;
+use Symfony\AI\Platform\Capability;
+use Symfony\AI\Platform\Contract\Normalizer\ModelContractNormalizer;
+use Symfony\AI\Platform\Message\Content\Image;
+use Symfony\AI\Platform\Model;
+
+use function Symfony\Component\String\u;
+
+final class ImageNormalizer extends ModelContractNormalizer
+{
+    /**
+     * @param Image $data
+     */
+    public function normalize(mixed $data, ?string $format = null, array $context = []): array
+    {
+        return [[
+            CollectionNormalizer::KEY_CONTENT => [[
+                'type' => 'image_base64',
+                'image_base64' => \sprintf(
+                    'data:%s;base64,%s',
+                    u($data->getFormat())->replace('jpg', 'jpeg'),
+                    $data->asBase64()
+                ),
+            ]],
+        ]];
+    }
+
+    protected function supportedDataClass(): string
+    {
+        return Image::class;
+    }
+
+    protected function supportsModel(Model $model): bool
+    {
+        return $model instanceof Voyage && $model->supports(Capability::INPUT_MULTIMODAL);
+    }
+}

src/platform/src/Bridge/Voyage/Contract/Multimodal/CollectionNormalizer.php

@@ -0,0 +1,44 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\Voyage\Contract\Multimodal;
+
+use Symfony\AI\Platform\Bridge\Voyage\Voyage;
+use Symfony\AI\Platform\Capability;
+use Symfony\AI\Platform\Contract\Normalizer\ModelContractNormalizer;
+use Symfony\AI\Platform\Message\Content\ImageUrl;
+use Symfony\AI\Platform\Model;
+
+final class ImageUrlNormalizer extends ModelContractNormalizer
+{
+    /**
+     * @param ImageUrl $data
+     */
+    public function normalize(mixed $data, ?string $format = null, array $context = []): array
+    {
+        return [[
+            CollectionNormalizer::KEY_CONTENT => [[
+                'type' => 'image_url',
+                'image_url' => $data->getUrl(),
+            ]],
+        ]];
+    }
+
+    protected function supportedDataClass(): string
+    {
+        return ImageUrl::class;
+    }
+
+    protected function supportsModel(Model $model): bool
+    {
+        return $model instanceof Voyage && $model->supports(Capability::INPUT_MULTIMODAL);
+    }
+}

src/platform/src/Bridge/Voyage/Contract/Multimodal/ImageNormalizer.php

@@ -0,0 +1,60 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\Voyage\Contract\Multimodal;
+
+use Symfony\AI\Platform\Bridge\Voyage\Voyage;
+use Symfony\AI\Platform\Capability;
+use Symfony\AI\Platform\Contract;
+use Symfony\AI\Platform\Message\Content\ContentInterface;
+use Symfony\Component\Serializer\Exception\ExceptionInterface;
+use Symfony\Component\Serializer\Normalizer\NormalizerAwareInterface;
+use Symfony\Component\Serializer\Normalizer\NormalizerAwareTrait;
+use Symfony\Component\Serializer\Normalizer\NormalizerInterface;
+
+final class MultimodalNormalizer implements NormalizerInterface, NormalizerAwareInterface
+{
+    use NormalizerAwareTrait;
+
+    /**
+     * @param ContentInterface[] $data
+     *
+     * @throws ExceptionInterface
+     */
+    public function normalize(mixed $data, ?string $format = null, array $context = []): array
+    {
+        return array_map(
+            function (ContentInterface $item) use ($format, $context) {
+                $normalized = $this->normalizer->normalize($item, $format, $context);
+
+                return array_pop($normalized);
+            },
+            $data
+        );
+    }
+
+    public function supportsNormalization(mixed $data, ?string $format = null, array $context = []): bool
+    {
+        $model = $context[Contract::CONTEXT_MODEL] ?? null;
+        if (!$model instanceof Voyage || !$model->supports(Capability::INPUT_MULTIMODAL)) {
+            return false;
+        }
+
+        return \is_array($data) && [] === array_filter($data, fn ($item) => !$item instanceof ContentInterface);
+    }
+
+    public function getSupportedTypes(?string $format): array
+    {
+        return [
+            'native-array' => true,
+        ];
+    }
+}

src/platform/src/Bridge/Voyage/Contract/Multimodal/ImageUrlNormalizer.php

@@ -0,0 +1,44 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\Voyage\Contract\Multimodal;
+
+use Symfony\AI\Platform\Bridge\Voyage\Voyage;
+use Symfony\AI\Platform\Capability;
+use Symfony\AI\Platform\Contract\Normalizer\ModelContractNormalizer;
+use Symfony\AI\Platform\Message\Content\Text;
+use Symfony\AI\Platform\Model;
+
+final class TextNormalizer extends ModelContractNormalizer
+{
+    /**
+     * @param Text $data
+     */
+    public function normalize(mixed $data, ?string $format = null, array $context = []): array
+    {
+        return [[
+            CollectionNormalizer::KEY_CONTENT => [[
+                'type' => 'text',
+                'text' => $data->getText(),
+            ]],
+        ]];
+    }
+
+    protected function supportedDataClass(): string
+    {
+        return Text::class;
+    }
+
+    protected function supportsModel(Model $model): bool
+    {
+        return $model instanceof Voyage && $model->supports(Capability::INPUT_MULTIMODAL);
+    }
+}