Merge pull request #47601 from nextcloud/enh/noid/use-taskprocessing-…

…in-old-managers

[TaskProcessing] Use taskprocessing in TextProcessing and STT managers

apps/settings/src/components/AdminAI.vue

@@ -50,24 +50,6 @@
 				</div>
 			</draggable>
 		</NcSettingsSection>
-		<NcSettingsSection :name="t('settings', 'Speech-To-Text')"
-			:description="t('settings', 'Speech-To-Text can be implemented by different apps. Here you can set which app should be used.')">
-			<template v-for="provider in sttProviders">
-				<NcCheckboxRadioSwitch :key="provider.class"
-					:checked.sync="settings['ai.stt_provider']"
-					:value="provider.class"
-					name="stt_provider"
-					type="radio"
-					@update:checked="saveChanges">
-					{{ provider.name }}
-				</NcCheckboxRadioSwitch>
-			</template>
-			<template v-if="!hasStt">
-				<NcNoteCard type="info">
-					{{ t('settings', 'None of your currently installed apps provide Speech-To-Text functionality') }}
-				</NcNoteCard>
-			</template>
-		</NcSettingsSection>
 		<NcSettingsSection :name="t('settings', 'Image generation')"
 			:description="t('settings', 'Image generation can be implemented by different apps. Here you can set which app should be used.')">
 			<template v-for="provider in text2imageProviders">
@@ -162,14 +144,19 @@ export default {
 		}
 	},
 	computed: {
-		hasStt() {
-			return this.sttProviders.length > 0
-		},
 		hasTextProcessing() {
 			return Object.keys(this.settings['ai.textprocessing_provider_preferences']).length > 0 && Array.isArray(this.textProcessingTaskTypes)
 		},
 		tpTaskTypes() {
-			return Object.keys(this.settings['ai.textprocessing_provider_preferences']).filter(type => !!this.getTextProcessingTaskType(type))
+			const builtinTextProcessingTypes = [
+				'\\OCP\\TextProcessing\\FreePromptTaskType',
+				'\\OCP\\TextProcessing\\HeadlineTaskType',
+				'\\OCP\\TextProcessing\\SummaryTaskType',
+				'\\OCP\\TextProcessing\\TopicsTaskType',
+			]
+			return Object.keys(this.settings['ai.textprocessing_provider_preferences'])
+				.filter(type => !!this.getTextProcessingTaskType(type))
+				.filter(type => !builtinTextProcessingTypes.includes(type))
 		},
 		hasText2ImageProviders() {
 		  return this.text2imageProviders.length > 0

lib/private/SpeechToText/SpeechToTextManager.php

@@ -24,6 +24,9 @@
 use OCP\SpeechToText\ISpeechToTextProvider;
 use OCP\SpeechToText\ISpeechToTextProviderWithId;
 use OCP\SpeechToText\ISpeechToTextProviderWithUserId;
+use OCP\TaskProcessing\IManager as ITaskProcessingManager;
+use OCP\TaskProcessing\Task;
+use OCP\TaskProcessing\TaskTypes\AudioToText;
 use Psr\Container\ContainerExceptionInterface;
 use Psr\Container\NotFoundExceptionInterface;
 use Psr\Log\LoggerInterface;
@@ -41,6 +44,7 @@ public function __construct(
 		private IJobList $jobList,
 		private IConfig $config,
 		private IUserSession $userSession,
+		private ITaskProcessingManager $taskProcessingManager,
 	) {
 	}
 
@@ -112,7 +116,30 @@ public function cancelScheduledFileTranscription(File $file, ?string $userId, st
 		}
 	}
 
-	public function transcribeFile(File $file): string {
+	public function transcribeFile(File $file, ?string $userId = null, string $appId = 'core'): string {
+		// try to run a TaskProcessing core:audio2text task
+		// this covers scheduling as well because OC\SpeechToText\TranscriptionJob calls this method
+		try {
+			if (isset($this->taskProcessingManager->getAvailableTaskTypes()['core:audio2text'])) {
+				$taskProcessingTask = new Task(
+					AudioToText::ID,
+					['input' => $file->getId()],
+					$appId,
+					$userId,
+					'from-SpeechToTextManager||' . $file->getId() . '||' . ($userId ?? '') . '||' . $appId,
+				);
+				$resultTask = $this->taskProcessingManager->runTask($taskProcessingTask);
+				if ($resultTask->getStatus() === Task::STATUS_SUCCESSFUL) {
+					$output = $resultTask->getOutput();
+					if (isset($output['output']) && is_string($output['output'])) {
+						return $output['output'];
+					}
+				}
+			}
+		} catch (Throwable $e) {
+			throw new RuntimeException('Failed to run a Speech-to-text job from STTManager with TaskProcessing for file ' . $file->getId(), 0, $e);
+		}
+
 		if (!$this->hasProviders()) {
 			throw new PreConditionNotMetException('No SpeechToText providers have been registered');
 		}

lib/private/SpeechToText/TranscriptionJob.php

@@ -63,7 +63,7 @@ protected function run($argument) {
 				);
 				return;
 			}
-			$result = $this->speechToTextManager->transcribeFile($file);
+			$result = $this->speechToTextManager->transcribeFile($file, $userId, $appId);
 			$this->eventDispatcher->dispatchTyped(
 				new TranscriptionSuccessfulEvent(
 					$fileId,

lib/private/TaskProcessing/Manager.php

@@ -87,9 +87,7 @@ public function __construct(
 		private IEventDispatcher $dispatcher,
 		IAppDataFactory $appDataFactory,
 		private IRootFolder $rootFolder,
-		private \OCP\TextProcessing\IManager $textProcessingManager,
 		private \OCP\TextToImage\IManager $textToImageManager,
-		private \OCP\SpeechToText\ISpeechToTextManager $speechToTextManager,
 		private IUserMountCache $userMountCache,
 		private IClientService $clientService,
 		private IAppManager $appManager,
@@ -98,8 +96,34 @@ public function __construct(
 	}
 
 
+	/**
+	 * This is almost a copy of textProcessingManager->getProviders
+	 * to avoid a dependency cycle between TextProcessingManager and TaskProcessingManager
+	 */
+	private function _getRawTextProcessingProviders(): array {
+		$context = $this->coordinator->getRegistrationContext();
+		if ($context === null) {
+			return [];
+		}
+
+		$providers = [];
+
+		foreach ($context->getTextProcessingProviders() as $providerServiceRegistration) {
+			$class = $providerServiceRegistration->getService();
+			try {
+				$providers[$class] = $this->serverContainer->get($class);
+			} catch (\Throwable $e) {
+				$this->logger->error('Failed to load Text processing provider ' . $class, [
+					'exception' => $e,
+				]);
+			}
+		}
+
+		return $providers;
+	}
+
 	private function _getTextProcessingProviders(): array {
-		$oldProviders = $this->textProcessingManager->getProviders();
+		$oldProviders = $this->_getRawTextProcessingProviders();
 		$newProviders = [];
 		foreach ($oldProviders as $oldProvider) {
 			$provider = new class($oldProvider) implements IProvider, ISynchronousProvider {
@@ -190,7 +214,7 @@ public function getOptionalOutputShapeEnumValues(): array {
 	 * @return ITaskType[]
 	 */
 	private function _getTextProcessingTaskTypes(): array {
-		$oldProviders = $this->textProcessingManager->getProviders();
+		$oldProviders = $this->_getRawTextProcessingProviders();
 		$newTaskTypes = [];
 		foreach ($oldProviders as $oldProvider) {
 			// These are already implemented in the TaskProcessing realm
@@ -344,12 +368,35 @@ public function getOptionalOutputShapeEnumValues(): array {
 		return $newProviders;
 	}
 
+	/**
+	 * This is almost a copy of SpeechToTextManager->getProviders
+	 * to avoid a dependency cycle between SpeechToTextManager and TaskProcessingManager
+	 */
+	private function _getRawSpeechToTextProviders(): array {
+		$context = $this->coordinator->getRegistrationContext();
+		if ($context === null) {
+			return [];
+		}
+		$providers = [];
+		foreach ($context->getSpeechToTextProviders() as $providerServiceRegistration) {
+			$class = $providerServiceRegistration->getService();
+			try {
+				$providers[$class] = $this->serverContainer->get($class);
+			} catch (NotFoundExceptionInterface|ContainerExceptionInterface|\Throwable $e) {
+				$this->logger->error('Failed to load SpeechToText provider ' . $class, [
+					'exception' => $e,
+				]);
+			}
+		}
+
+		return $providers;
+	}
 
 	/**
 	 * @return IProvider[]
 	 */
 	private function _getSpeechToTextProviders(): array {
-		$oldProviders = $this->speechToTextManager->getProviders();
+		$oldProviders = $this->_getRawSpeechToTextProviders();
 		$newProviders = [];
 		foreach ($oldProviders as $oldProvider) {
 			$newProvider = new class($oldProvider, $this->rootFolder, $this->appData) implements IProvider, ISynchronousProvider {

lib/private/TextProcessing/Manager.php

@@ -20,13 +20,22 @@
 use OCP\IConfig;
 use OCP\IServerContainer;
 use OCP\PreConditionNotMetException;
+use OCP\TaskProcessing\IManager as TaskProcessingIManager;
+use OCP\TaskProcessing\TaskTypes\TextToText;
+use OCP\TaskProcessing\TaskTypes\TextToTextHeadline;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
+use OCP\TaskProcessing\TaskTypes\TextToTextTopics;
 use OCP\TextProcessing\Exception\TaskFailureException;
+use OCP\TextProcessing\FreePromptTaskType;
+use OCP\TextProcessing\HeadlineTaskType;
 use OCP\TextProcessing\IManager;
 use OCP\TextProcessing\IProvider;
 use OCP\TextProcessing\IProviderWithExpectedRuntime;
 use OCP\TextProcessing\IProviderWithId;
+use OCP\TextProcessing\SummaryTaskType;
 use OCP\TextProcessing\Task;
 use OCP\TextProcessing\Task as OCPTask;
+use OCP\TextProcessing\TopicsTaskType;
 use Psr\Log\LoggerInterface;
 use RuntimeException;
 use Throwable;
@@ -42,6 +51,7 @@ public function __construct(
 		private IJobList $jobList,
 		private TaskMapper $taskMapper,
 		private IConfig $config,
+		private TaskProcessingIManager $taskProcessingManager,
 	) {
 	}
 
@@ -98,6 +108,55 @@ public function canHandleTask(OCPTask $task): bool {
 	 * @inheritDoc
 	 */
 	public function runTask(OCPTask $task): string {
+		// try to run a task processing task if possible
+		$taskTypeClass = $task->getType();
+		$taskProcessingCompatibleTaskTypes = [
+			FreePromptTaskType::class => TextToText::ID,
+			HeadlineTaskType::class => TextToTextHeadline::ID,
+			SummaryTaskType::class => TextToTextSummary::ID,
+			TopicsTaskType::class => TextToTextTopics::ID,
+		];
+		if (isset($taskProcessingCompatibleTaskTypes[$taskTypeClass]) && isset($this->taskProcessingManager->getAvailableTaskTypes()[$taskProcessingCompatibleTaskTypes[$taskTypeClass]])) {
+			try {
+				$taskProcessingTaskTypeId = $taskProcessingCompatibleTaskTypes[$taskTypeClass];
+				$taskProcessingTask = new \OCP\TaskProcessing\Task(
+					$taskProcessingTaskTypeId,
+					['input' => $task->getInput()],
+					$task->getAppId(),
+					$task->getUserId(),
+					$task->getIdentifier(),
+				);
+
+				$task->setStatus(OCPTask::STATUS_RUNNING);
+				if ($task->getId() === null) {
+					$taskEntity = $this->taskMapper->insert(DbTask::fromPublicTask($task));
+					$task->setId($taskEntity->getId());
+				} else {
+					$this->taskMapper->update(DbTask::fromPublicTask($task));
+				}
+				$this->logger->debug('Running a TextProcessing (' . $taskTypeClass . ') task with TaskProcessing');
+				$taskProcessingResultTask = $this->taskProcessingManager->runTask($taskProcessingTask);
+				if ($taskProcessingResultTask->getStatus() === \OCP\TaskProcessing\Task::STATUS_SUCCESSFUL) {
+					$output = $taskProcessingResultTask->getOutput();
+					if (isset($output['output']) && is_string($output['output'])) {
+						$task->setOutput($output['output']);
+						$task->setStatus(OCPTask::STATUS_SUCCESSFUL);
+						$this->taskMapper->update(DbTask::fromPublicTask($task));
+						return $output['output'];
+					}
+				}
+			} catch (\Throwable $e) {
+				$this->logger->error('TextProcessing to TaskProcessing failed', ['exception' => $e]);
+				$task->setStatus(OCPTask::STATUS_FAILED);
+				$this->taskMapper->update(DbTask::fromPublicTask($task));
+				throw new TaskFailureException('TextProcessing to TaskProcessing failed: ' . $e->getMessage(), 0, $e);
+			}
+			$task->setStatus(OCPTask::STATUS_FAILED);
+			$this->taskMapper->update(DbTask::fromPublicTask($task));
+			throw new TaskFailureException('Could not run task');
+		}
+
+		// try to run the text processing task
 		if (!$this->canHandleTask($task)) {
 			throw new PreConditionNotMetException('No text processing provider is installed that can handle this task');
 		}

lib/public/SpeechToText/ISpeechToTextManager.php

@@ -59,11 +59,13 @@ public function cancelScheduledFileTranscription(File $file, ?string $userId, st
 
 	/**
 	 * @param File $file The media file to transcribe
+	 * @param ?string $userId The user that triggered this request
+	 * @param string $appId The app that triggered this request
 	 * @returns string The transcription of the passed media file
 	 * @throws PreConditionNotMetException If no provider was registered but this method was still called
 	 * @throws InvalidArgumentException If the file could not be found or is not of a supported type
 	 * @throws RuntimeException If the transcription failed for other reasons
 	 * @since 27.0.0
 	 */
-	public function transcribeFile(File $file): string;
+	public function transcribeFile(File $file, ?string $userId, string $appId): string;
 }

tests/lib/TaskProcessing/TaskProcessingTest.php

@@ -3,7 +3,7 @@
  * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
-namespace Test\TextProcessing;
+namespace Test\TaskProcessing;
 
 use OC\AppFramework\Bootstrap\Coordinator;
 use OC\AppFramework\Bootstrap\RegistrationContext;
@@ -26,7 +26,6 @@
 use OCP\IServerContainer;
 use OCP\IUser;
 use OCP\IUserManager;
-use OCP\SpeechToText\ISpeechToTextManager;
 use OCP\TaskProcessing\EShapeType;
 use OCP\TaskProcessing\Events\TaskFailedEvent;
 use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
@@ -450,15 +449,6 @@ protected function setUp(): void {
 
 		$this->eventDispatcher = $this->createMock(IEventDispatcher::class);
 
-		$textProcessingManager = new \OC\TextProcessing\Manager(
-			$this->serverContainer,
-			$this->coordinator,
-			\OC::$server->get(LoggerInterface::class),
-			$this->jobList,
-			\OC::$server->get(\OC\TextProcessing\Db\TaskMapper::class),
-			\OC::$server->get(IConfig::class),
-		);
-
 		$text2imageManager = new \OC\TextToImage\Manager(
 			$this->serverContainer,
 			$this->coordinator,
@@ -481,9 +471,7 @@ protected function setUp(): void {
 			$this->eventDispatcher,
 			\OC::$server->get(IAppDataFactory::class),
 			\OC::$server->get(IRootFolder::class),
-			$textProcessingManager,
 			$text2imageManager,
-			\OC::$server->get(ISpeechToTextManager::class),
 			$this->userMountCache,
 			\OC::$server->get(IClientService::class),
 			\OC::$server->get(IAppManager::class),

tests/lib/TextProcessing/TextProcessingTest.php

@@ -86,6 +86,9 @@ public function getTaskType(): string {
 	}
 }
 
+/**
+ * @group DB
+ */
 class TextProcessingTest extends \Test\TestCase {
 	private IManager $manager;
 	private Coordinator $coordinator;
@@ -176,7 +179,8 @@ protected function setUp(): void {
 			\OC::$server->get(LoggerInterface::class),
 			$this->jobList,
 			$this->taskMapper,
-			$config
+			$config,
+			\OC::$server->get(\OCP\TaskProcessing\IManager::class),
 		);
 	}