Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions lib/Service/FileFormatDetector.php

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: image_type_to_extension(getimagesizefromstring($data)[2], false) can return the extension as well, instead of doing it manually

Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCA\OpenAi\Service;

class FileFormatDetector {
/**
* Detects the file extension of binary image data from its magic bytes
*
* @param string $data
* @return string The detected file extension, or '' if the format is unknown
*/
public static function getImageExtension(string $data): string {
if (str_starts_with($data, "\xFF\xD8\xFF")) {
return 'jpeg';
}
if (str_starts_with($data, "\x89PNG\r\n\x1A\n")) {
return 'png';
}
if (str_starts_with($data, 'GIF8')) {
return 'gif';
}
if (str_starts_with($data, 'RIFF') && substr($data, 8, 4) === 'WEBP') {
return 'webp';
}
return '';
}

/**
* Detects the file extension of binary audio data from its magic bytes
*
* @param string $data
* @return string The detected file extension, or '' if the format is unknown
*/
public static function getAudioExtension(string $data): string {
if (str_starts_with($data, 'ID3')) {
return 'mp3';
}
if (str_starts_with($data, 'RIFF') && substr($data, 8, 4) === 'WAVE') {
return 'wav';
}
if (str_starts_with($data, 'OggS')) {
return 'ogg';
}
if (str_starts_with($data, 'fLaC')) {
return 'flac';
}
if (strlen($data) >= 8 && substr($data, 4, 4) === 'ftyp') {
// MP4 container, brand distinguishes Apple audio files
return substr($data, 8, 4) === 'M4A ' ? 'm4a' : 'mp4';
}
if (strlen($data) >= 2 && ord($data[0]) === 0xFF && (ord($data[1]) & 0xE0) === 0xE0) {
$b2 = ord($data[1]);
// MPEG version 0b01 is reserved, neither valid MPEG audio nor ADTS
if (($b2 & 0x18) === 0x08) {
return '';
}
// zero layer bits identify ADTS AAC, never valid MPEG audio
if (($b2 & 0x06) === 0) {
return 'aac';
}
return 'mp3';
}
return '';
}
}
4 changes: 3 additions & 1 deletion lib/TaskProcessing/AudioToAudioChatProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@

namespace OCA\OpenAi\TaskProcessing;

use OCA\OpenAi\Service\FileFormatDetector;
use OCA\OpenAi\Service\OpenAiAPIService;
use OCA\OpenAi\Service\ServiceConfig;
use OCP\Files\File;
use OCP\IL10N;
use OCP\TaskProcessing\EShapeType;
use OCP\TaskProcessing\Exception\ProcessingException;
use OCP\TaskProcessing\Exception\UserFacingProcessingException;
use OCP\TaskProcessing\FileShaped;
use OCP\TaskProcessing\ISynchronousProvider;
use OCP\TaskProcessing\ShapeDescriptor;
use OCP\TaskProcessing\ShapeEnumValue;
Expand Down Expand Up @@ -211,7 +213,7 @@ public function process(?string $userId, array $input, callable $reportProgress)
$result['audio_expires_at'] = $message['audio']['expires_at'];
}
}
$result['output'] = $output;
$result['output'] = new FileShaped(EShapeType::Audio, $output, FileFormatDetector::getAudioExtension($output) ?: 'mp3');
$result['output_transcript'] = $textResponse;

// the transcript of the input is part of the task output
Expand Down
4 changes: 3 additions & 1 deletion lib/TaskProcessing/AudioToAudioTranslateProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

use Exception;
use OCA\OpenAi\AppInfo\Application;
use OCA\OpenAi\Service\FileFormatDetector;
use OCA\OpenAi\Service\OpenAiAPIService;
use OCA\OpenAi\Service\ServiceConfig;
use OCA\OpenAi\Service\TranslateService;
Expand All @@ -22,6 +23,7 @@
use OCP\TaskProcessing\EShapeType;
use OCP\TaskProcessing\Exception\ProcessingException;
use OCP\TaskProcessing\Exception\UserFacingProcessingException;
use OCP\TaskProcessing\FileShaped;
use OCP\TaskProcessing\IProvider;
use OCP\TaskProcessing\ISynchronousOptionsAwareProvider;
use OCP\TaskProcessing\ShapeDescriptor;
Expand Down Expand Up @@ -296,7 +298,7 @@ public function process(
// Translation
return [
'text_input' => $transcription . $watermarkSuffix,
'audio_output' => $translatedAudio,
'audio_output' => new FileShaped(EShapeType::Audio, $translatedAudio, FileFormatDetector::getAudioExtension($translatedAudio) ?: 'mp3'),
'text_output' => $translatedText . $watermarkSuffix,
];
}
Expand Down
4 changes: 3 additions & 1 deletion lib/TaskProcessing/ImageToImageProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
namespace OCA\OpenAi\TaskProcessing;

use OCA\OpenAi\AppInfo\Application;
use OCA\OpenAi\Service\FileFormatDetector;
use OCA\OpenAi\Service\OpenAiAPIService;
use OCA\OpenAi\Service\ServiceConfig;
use OCA\OpenAi\Service\WatermarkingService;
Expand All @@ -19,6 +20,7 @@
use OCP\TaskProcessing\EShapeType;
use OCP\TaskProcessing\Exception\ProcessingException;
use OCP\TaskProcessing\Exception\UserFacingProcessingException;
use OCP\TaskProcessing\FileShaped;
use OCP\TaskProcessing\ISynchronousOptionsAwareProvider;
use OCP\TaskProcessing\ShapeDescriptor;
use OCP\TaskProcessing\SynchronousProviderOptions;
Expand Down Expand Up @@ -221,7 +223,7 @@ function ($carry, $file) {
$image = $includeWatermark ? $this->watermarkingService->markImage($image) : $image;
$endTime = time();
$this->openAiAPIService->updateExpImgProcessingTime($endTime - $startTime, $this->service);
return ['output' => $image];
return ['output' => new FileShaped(EShapeType::Image, $image, FileFormatDetector::getImageExtension($image))];
} catch (UserFacingProcessingException $e) {
throw $e;
} catch (\Throwable $e) {
Expand Down
8 changes: 5 additions & 3 deletions lib/TaskProcessing/TextToImageProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
namespace OCA\OpenAi\TaskProcessing;

use OCA\OpenAi\AppInfo\Application;
use OCA\OpenAi\Service\FileFormatDetector;
use OCA\OpenAi\Service\OpenAiAPIService;
use OCA\OpenAi\Service\ServiceConfig;
use OCA\OpenAi\Service\WatermarkingService;
Expand All @@ -18,6 +19,7 @@
use OCP\TaskProcessing\EShapeType;
use OCP\TaskProcessing\Exception\ProcessingException;
use OCP\TaskProcessing\Exception\UserFacingProcessingException;
use OCP\TaskProcessing\FileShaped;
use OCP\TaskProcessing\ISynchronousWatermarkingProvider;
use OCP\TaskProcessing\ShapeDescriptor;
use OCP\TaskProcessing\TaskTypes\TextToImage;
Expand Down Expand Up @@ -159,16 +161,16 @@ public function process(?string $userId, array $input, callable $reportProgress,
$imageResponse = $client->get($url, $requestOptions);
$image = $imageResponse->getBody();
$image = $includeWatermark ? $this->watermarkingService->markImage($image) : $image;
$output['images'][] = $image;
$output['images'][] = new FileShaped(EShapeType::Image, $image, FileFormatDetector::getImageExtension($image));
}
foreach ($b64s as $b64) {
$imagePayload = base64_decode($b64);
$image = $includeWatermark ? $this->watermarkingService->markImage($imagePayload) : $imagePayload;
$output['images'][] = $image;
$output['images'][] = new FileShaped(EShapeType::Image, $image, FileFormatDetector::getImageExtension($image));
}
$endTime = time();
$this->openAiAPIService->updateExpImgProcessingTime($endTime - $startTime, $this->service);
/** @var array<string, list<numeric|string>|numeric|string> $output */
/** @var array<string, list<numeric|string|FileShaped>|numeric|string|FileShaped> $output */
return $output;
} catch (UserFacingProcessingException $e) {
throw $e;
Expand Down
4 changes: 3 additions & 1 deletion lib/TaskProcessing/TextToSpeechProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@

namespace OCA\OpenAi\TaskProcessing;

use OCA\OpenAi\Service\FileFormatDetector;
use OCA\OpenAi\Service\OpenAiAPIService;
use OCA\OpenAi\Service\ServiceConfig;
use OCA\OpenAi\Service\WatermarkingService;
use OCP\IL10N;
use OCP\TaskProcessing\EShapeType;
use OCP\TaskProcessing\Exception\ProcessingException;
use OCP\TaskProcessing\Exception\UserFacingProcessingException;
use OCP\TaskProcessing\FileShaped;
use OCP\TaskProcessing\ISynchronousWatermarkingProvider;
use OCP\TaskProcessing\ShapeDescriptor;
use OCP\TaskProcessing\ShapeEnumValue;
Expand Down Expand Up @@ -141,7 +143,7 @@ public function process(?string $userId, array $input, callable $reportProgress,
}
$audio = $includeWatermark ? $this->watermarkingService->markAudio($apiResponse['body']) : $apiResponse['body'];

return ['speech' => $audio];
return ['speech' => new FileShaped(EShapeType::Audio, $audio, FileFormatDetector::getAudioExtension($audio) ?: 'mp3')];
} catch (UserFacingProcessingException $e) {
throw $e;
} catch (\Throwable $e) {
Expand Down
1 change: 1 addition & 0 deletions psalm.xml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
<file name="tests/stubs/oc_hooks_emitter.php" />
<file name="tests/stubs/ocp_task_processing_i_provider.php" />
<file name="tests/stubs/ocp_task_processing_isynchronous_provider.php" />
<file name="tests/stubs/ocp_task_processing_isynchronous_watermarking_provider.php" />
<file name="tests/stubs/ocp_task_processing_synchronous_provider_options.php" />
<file name="tests/stubs/ocp_task_processing_isynchronous_options_provider.php" />
</stubs>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCP\TaskProcessing;

/**
* The vendored nextcloud/ocp snapshot misses the FileShaped return type
* in the @psalm-return annotation of this deprecated interface
*
* @since 33.0.0
*/
interface ISynchronousWatermarkingProvider extends ISynchronousProvider {
public function process(?string $userId, array $input, callable $reportProgress, bool $includeWatermark = true): array;
}
9 changes: 7 additions & 2 deletions tests/unit/Providers/ImageToImageProviderSizeTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
use OCP\Http\Client\IClientService;
use OCP\IL10N;
use OCP\TaskProcessing\Exception\UserFacingProcessingException;
use OCP\TaskProcessing\FileShaped;
use PHPUnit\Framework\MockObject\MockObject;
use PHPUnit\Framework\TestCase;
use Psr\Log\LoggerInterface;
Expand Down Expand Up @@ -88,7 +89,9 @@ public function testGptImage1AcceptsValidSize(string $size): void {
->willReturn(['data' => [['b64_json' => base64_encode('img')]]]);

$result = $provider->process('user', $this->validInput(['size' => $size]), static fn () => null);
$this->assertSame(['output' => 'img'], $result);
$this->assertInstanceOf(FileShaped::class, $result['output']);
$this->assertSame('img', $result['output']->getData());
$this->assertSame('', $result['output']->getExtension());
}

/**
Expand Down Expand Up @@ -157,7 +160,9 @@ public function testIonosAcceptsValidSize(string $size): void {
->willReturn(['data' => [['b64_json' => base64_encode('img')]]]);

$result = $provider->process('user', $this->validInput(['size' => $size]), static fn () => null);
$this->assertSame(['output' => 'img'], $result);
$this->assertInstanceOf(FileShaped::class, $result['output']);
$this->assertSame('img', $result['output']->getData());
$this->assertSame('', $result['output']->getExtension());
}

/**
Expand Down
16 changes: 14 additions & 2 deletions tests/unit/Providers/OpenAiProviderTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
use OCP\Http\Client\IClientService;
use OCP\IAppConfig;
use OCP\ICacheFactory;
use OCP\TaskProcessing\EShapeType;
use OCP\TaskProcessing\FileShaped;
use OCP\TaskProcessing\SynchronousProviderOptions;
use OCP\TaskProcessing\TaskTypes\TextToTextReformatParagraphs;
use PHPUnit\Framework\MockObject\MockObject;
Expand Down Expand Up @@ -1015,7 +1017,9 @@ public function testAudioToAudioTranslateProvider(): void {
$this->assertArrayHasKey('audio_output', $result);
$this->assertEquals($transcribedText, $result['text_input']);
$this->assertEquals($translatedText, $result['text_output']);
$this->assertEquals($ttsResponse, $result['audio_output']);
$this->assertInstanceOf(FileShaped::class, $result['audio_output']);
$this->assertEquals($ttsResponse, $result['audio_output']->getData());
$this->assertSame('mp3', $result['audio_output']->getExtension());

$this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1);
}
Expand Down Expand Up @@ -1059,6 +1063,8 @@ public function testTextToSpeechProvider(): void {

$result = $TTSProvider->process(self::TEST_USER1, ['input' => $inputText], fn () => true);
$this->assertArrayHasKey('speech', $result);
$this->assertInstanceOf(FileShaped::class, $result['speech']);
$this->assertSame('mp3', $result['speech']->getExtension());

// Check that token usage is logged properly (should be 21 characters)
$usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_SPEECH);
Expand Down Expand Up @@ -1114,6 +1120,9 @@ public function testTextToImageProvider(): void {
$result = $TextToImageProvider->process(self::TEST_USER1, ['input' => $inputText, 'numberOfImages' => 1], fn () => true);
$this->assertArrayHasKey('images', $result);
$this->assertArrayHasKey(0, $result['images']);
$this->assertInstanceOf(FileShaped::class, $result['images'][0]);
$this->assertEquals(EShapeType::Image, $result['images'][0]->getShapeType());
$this->assertSame('jpeg', $result['images'][0]->getExtension());

// Check that token usage is logged properly (should be 21 characters)
$usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_IMAGE);
Expand Down Expand Up @@ -1187,7 +1196,10 @@ public function testImageToImageProvider(): void {
fn () => true,
);
$this->assertArrayHasKey('output', $result);
$this->assertEquals($inputImage, $result['output']);
$this->assertInstanceOf(FileShaped::class, $result['output']);
$this->assertEquals(EShapeType::Image, $result['output']->getShapeType());
$this->assertSame('jpeg', $result['output']->getExtension());
$this->assertEquals($inputImage, $result['output']->getData());

$usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_IMAGE);
$this->assertEquals(1, $usage);
Expand Down
Loading