Merge pull request #48758 from nextcloud/feat/files/chunked-upload-co…

…nfig-capabilities
nextcloud · Oct 21, 2024 · 432ea6e · 432ea6e
2 parents 4d8d11d + ba3d67d
commit 432ea6e
Show file tree

Hide file tree

Showing 9 changed files with 115 additions and 4 deletions.
diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml
@@ -10,7 +10,7 @@
 	<name>Files</name>
 	<summary>File Management</summary>
 	<description>File Management</description>
-	<version>2.3.0</version>
+	<version>2.3.1</version>
 	<licence>agpl</licence>
 	<author>John Molakvoæ</author>
 	<author>Robin Appelman</author>

diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php
@@ -67,9 +67,11 @@
     'OCA\\Files\\Listener\\SyncLivePhotosListener' => $baseDir . '/../lib/Listener/SyncLivePhotosListener.php',
     'OCA\\Files\\Migration\\Version11301Date20191205150729' => $baseDir . '/../lib/Migration/Version11301Date20191205150729.php',
     'OCA\\Files\\Migration\\Version12101Date20221011153334' => $baseDir . '/../lib/Migration/Version12101Date20221011153334.php',
+    'OCA\\Files\\Migration\\Version2003Date20241021095629' => $baseDir . '/../lib/Migration/Version2003Date20241021095629.php',
     'OCA\\Files\\Notification\\Notifier' => $baseDir . '/../lib/Notification/Notifier.php',
     'OCA\\Files\\ResponseDefinitions' => $baseDir . '/../lib/ResponseDefinitions.php',
     'OCA\\Files\\Search\\FilesSearchProvider' => $baseDir . '/../lib/Search/FilesSearchProvider.php',
+    'OCA\\Files\\Service\\ChunkedUploadConfig' => $baseDir . '/../lib/Service/ChunkedUploadConfig.php',
     'OCA\\Files\\Service\\DirectEditingService' => $baseDir . '/../lib/Service/DirectEditingService.php',
     'OCA\\Files\\Service\\LivePhotosService' => $baseDir . '/../lib/Service/LivePhotosService.php',
     'OCA\\Files\\Service\\OwnershipTransferService' => $baseDir . '/../lib/Service/OwnershipTransferService.php',

diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php
@@ -82,9 +82,11 @@ class ComposerStaticInitFiles
         'OCA\\Files\\Listener\\SyncLivePhotosListener' => __DIR__ . '/..' . '/../lib/Listener/SyncLivePhotosListener.php',
         'OCA\\Files\\Migration\\Version11301Date20191205150729' => __DIR__ . '/..' . '/../lib/Migration/Version11301Date20191205150729.php',
         'OCA\\Files\\Migration\\Version12101Date20221011153334' => __DIR__ . '/..' . '/../lib/Migration/Version12101Date20221011153334.php',
+        'OCA\\Files\\Migration\\Version2003Date20241021095629' => __DIR__ . '/..' . '/../lib/Migration/Version2003Date20241021095629.php',
         'OCA\\Files\\Notification\\Notifier' => __DIR__ . '/..' . '/../lib/Notification/Notifier.php',
         'OCA\\Files\\ResponseDefinitions' => __DIR__ . '/..' . '/../lib/ResponseDefinitions.php',
         'OCA\\Files\\Search\\FilesSearchProvider' => __DIR__ . '/..' . '/../lib/Search/FilesSearchProvider.php',
+        'OCA\\Files\\Service\\ChunkedUploadConfig' => __DIR__ . '/..' . '/../lib/Service/ChunkedUploadConfig.php',
         'OCA\\Files\\Service\\DirectEditingService' => __DIR__ . '/..' . '/../lib/Service/DirectEditingService.php',
         'OCA\\Files\\Service\\LivePhotosService' => __DIR__ . '/..' . '/../lib/Service/LivePhotosService.php',
         'OCA\\Files\\Service\\OwnershipTransferService' => __DIR__ . '/..' . '/../lib/Service/OwnershipTransferService.php',

diff --git a/apps/files/lib/App.php b/apps/files/lib/App.php
@@ -8,6 +8,7 @@
 namespace OCA\Files;
 
 use OC\NavigationManager;
+use OCA\Files\Service\ChunkedUploadConfig;
 use OCP\App\IAppManager;
 use OCP\IConfig;
 use OCP\IGroupManager;
@@ -44,9 +45,8 @@ public static function getNavigationManager(): INavigationManager {
 	public static function extendJsConfig($settings): void {
 		$appConfig = json_decode($settings['array']['oc_appconfig'], true);
 
-		$maxChunkSize = (int)Server::get(IConfig::class)->getAppValue('files', 'max_chunk_size', (string)(100 * 1024 * 1024));
 		$appConfig['files'] = [
-			'max_chunk_size' => $maxChunkSize
+			'max_chunk_size' => ChunkedUploadConfig::getMaxChunkSize(),
 		];
 
 		$settings['array']['oc_appconfig'] = json_encode($appConfig);

diff --git a/apps/files/lib/Capabilities.php b/apps/files/lib/Capabilities.php
@@ -8,6 +8,7 @@
 namespace OCA\Files;
 
 use OC\Files\FilenameValidator;
+use OCA\Files\Service\ChunkedUploadConfig;
 use OCP\Capabilities\ICapability;
 
 class Capabilities implements ICapability {
@@ -20,7 +21,7 @@ public function __construct(
 	/**
 	 * Return this classes capabilities
 	 *
-	 * @return array{files: array{'$comment': ?string, bigfilechunking: bool, blacklisted_files: array<mixed>, forbidden_filenames: list<string>, forbidden_filename_basenames: list<string>, forbidden_filename_characters: list<string>, forbidden_filename_extensions: list<string>}}
+	 * @return array{files: array{'$comment': ?string, bigfilechunking: bool, blacklisted_files: array<mixed>, forbidden_filenames: list<string>, forbidden_filename_basenames: list<string>, forbidden_filename_characters: list<string>, forbidden_filename_extensions: list<string>, chunked_upload: array{max_size: int, max_parallel_count: int}}}
 	 */
 	public function getCapabilities(): array {
 		return [
@@ -33,6 +34,10 @@ public function getCapabilities(): array {
 				'forbidden_filename_extensions' => $this->filenameValidator->getForbiddenExtensions(),
 
 				'bigfilechunking' => true,
+				'chunked_upload' => [
+					'max_size' => ChunkedUploadConfig::getMaxChunkSize(),
+					'max_parallel_count' => ChunkedUploadConfig::getMaxParallelCount(),
+				],
 			],
 		];
 	}

diff --git a/apps/files/lib/Migration/Version2003Date20241021095629.php b/apps/files/lib/Migration/Version2003Date20241021095629.php
@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Files\Migration;
+
+use Closure;
+use OCA\Files\Service\ChunkedUploadConfig;
+use OCP\DB\ISchemaWrapper;
+use OCP\IConfig;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+use OCP\Server;
+
+class Version2003Date20241021095629 extends SimpleMigrationStep {
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 */
+	public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
+		$maxChunkSize = Server::get(IConfig::class)->getAppValue('files', 'max_chunk_size');
+		if ($maxChunkSize === '') {
+			// Skip if no value was configured before
+			return;
+		}
+
+		ChunkedUploadConfig::setMaxChunkSize((int)$maxChunkSize);
+		Server::get(IConfig::class)->deleteAppValue('files', 'max_chunk_size');
+	}
+}
diff --git a/apps/files/lib/Service/ChunkedUploadConfig.php b/apps/files/lib/Service/ChunkedUploadConfig.php
@@ -0,0 +1,30 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Files\Service;
+
+use OCP\IConfig;
+use OCP\Server;
+
+class ChunkedUploadConfig {
+	private const KEY_MAX_SIZE = 'files.chunked_upload.max_size';
+	private const KEY_MAX_PARALLEL_COUNT = 'files.chunked_upload.max_parallel_count';
+
+	public static function getMaxChunkSize(): int {
+		return Server::get(IConfig::class)->getSystemValueInt(self::KEY_MAX_SIZE, 100 * 1024 * 1024);
+	}
+
+	public static function setMaxChunkSize(int $maxChunkSize): void {
+		Server::get(IConfig::class)->setSystemValue(self::KEY_MAX_SIZE, $maxChunkSize);
+	}
+
+	public static function getMaxParallelCount(): int {
+		return Server::get(IConfig::class)->getSystemValueInt(self::KEY_MAX_PARALLEL_COUNT, 5);
+	}
+}
diff --git a/apps/files/openapi.json b/apps/files/openapi.json
@@ -36,6 +36,7 @@
                             "forbidden_filename_basenames",
                             "forbidden_filename_characters",
                             "forbidden_filename_extensions",
+                            "chunked_upload",
                             "directEditing"
                         ],
                         "properties": {
@@ -76,6 +77,23 @@
                                     "type": "string"
                                 }
                             },
+                            "chunked_upload": {
+                                "type": "object",
+                                "required": [
+                                    "max_size",
+                                    "max_parallel_count"
+                                ],
+                                "properties": {
+                                    "max_size": {
+                                        "type": "integer",
+                                        "format": "int64"
+                                    },
+                                    "max_parallel_count": {
+                                        "type": "integer",
+                                        "format": "int64"
+                                    }
+                                }
+                            },
                             "directEditing": {
                                 "type": "object",
                                 "required": [

diff --git a/config/config.sample.php b/config/config.sample.php
@@ -2556,4 +2556,22 @@
 	'/bin',
 	'/opt/bin',
 ],
+
+/**
+ * The maximum chunk size to use for chunked uploads.
+ * A bigger chunk size results in higher throughput, but above 100 MiB there are only diminishing returns,
+ * while services like Cloudflare already limit to 100 MiB.
+ *
+ * Defaults to 100 MiB.
+ */
+'files.chunked_upload.max_size' => 100 * 1024 * 1024,
+
+/**
+ * The maximum number of chunks uploaded in parallel during chunked uploads.
+ * A bigger count results in higher throughput, but will also consume more server workers,
+ * while the improvements diminish.
+ *
+ * Defaults to 5.
+ */
+'files.chunked_upload.max_parallel_count' => 5,
 ];