support multiple images

Signed-off-by: Lukas Schaefer <[email protected]>

Author: lukasdotcom

lib/TaskProcessing/AnalyzeImageProvider.php

@@ -110,27 +110,45 @@ public function process(?string $userId, array $input, callable $reportProgress)
 			throw new RuntimeException('Must support chat completion endpoint');
 		}
 
-		if (!isset($input['image']) || !$input['image'] instanceof File || !$input['image']->isReadable()) {
-			throw new RuntimeException('Invalid input file');
-		}
+		$history = [];
 
-		$inputFile = base64_encode(stream_get_contents($input['image']->fopen('rb')));
-		$fileType = $input['image']->getMimeType();
-		if (!str_starts_with($fileType, 'image/')) {
-			throw new RuntimeException('Invalid input file type ' . $fileType);
+		if (!isset($input['image']) || !is_array($input['image'])) {
+			throw new RuntimeException('Invalid file list');
 		}
-		if ($this->openAiAPIService->isUsingOpenAi()) {
-			$validFileTypes = [
-				'image/jpeg',
-				'image/png',
-				'image/gif',
-				'image/webp',
-			];
-			if (!in_array($fileType, $validFileTypes)) {
-				throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType);
+		foreach ($input['image'] as $image) {
+			if (!$image instanceof File || !$image->isReadable()) {
+				throw new RuntimeException('Invalid input file');
+			}
+			$inputFile = base64_encode(stream_get_contents($image->fopen('rb')));
+			$fileType = $image->getMimeType();
+			if (!str_starts_with($fileType, 'image/')) {
+				throw new RuntimeException('Invalid input file type ' . $fileType);
 			}
+			if ($this->openAiAPIService->isUsingOpenAi()) {
+				$validFileTypes = [
+					'image/jpeg',
+					'image/png',
+					'image/gif',
+					'image/webp',
+				];
+				if (!in_array($fileType, $validFileTypes)) {
+					throw new RuntimeException('Invalid input file type for OpenAI ' . $fileType);
+				}
+			}
+			$history[] = json_encode([
+				'role' => 'user',
+				'content' => [
+					[
+						'type' => 'image_url',
+						'image_url' => [
+							'url' => 'data:' . $fileType . ';base64,' . $inputFile,
+						],
+					],
+				],
+			]);
 		}
 
+
 		if (!isset($input['input']) || !is_string($input['input'])) {
 			throw new RuntimeException('Invalid prompt');
 		}
@@ -148,20 +166,8 @@ public function process(?string $userId, array $input, callable $reportProgress)
 		}
 
 		try {
-			$systemPrompt = 'Take the user\'s question and answer it based on the provided image. Ensure that the answer matches the language of the user\'s question.';
-			$completion = $this->openAiAPIService->createChatCompletion($userId, $model, $prompt, $systemPrompt, [
-				json_encode([
-					'role' => 'user',
-					'content' => [
-						[
-							'type' => 'image_url',
-							'image_url' => [
-								'url' => 'data:' . $fileType . ';base64,' . $inputFile,
-							],
-						],
-					],
-				])
-			], 1, $maxTokens);
+			$systemPrompt = 'Take the user\'s question and answer it based on the provided images. Ensure that the answer matches the language of the user\'s question.';
+			$completion = $this->openAiAPIService->createChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens);
 			$completion = $completion['messages'];
 
 			if (count($completion) > 0) {

lib/TaskProcessing/AnalyzeImageTaskType.php

@@ -50,9 +50,9 @@ public function getId(): string {
 	public function getInputShape(): array {
 		return [
 			'image' => new ShapeDescriptor(
-				$this->l->t('Image'),
-				$this->l->t('Image to ask a question about'),
-				EShapeType::Image,
+				$this->l->t('Images'),
+				$this->l->t('Images to ask a question about'),
+				EShapeType::ListOfImages,
 			),
 			'input' => new ShapeDescriptor(
 				$this->l->t('Question'),