[stable26] enh/import from deck

.github/workflows/integration.yml

@@ -71,7 +71,7 @@ jobs:
           path: apps/${{ env.APP_NAME }}
 
       - name: Set up php ${{ matrix.php-versions }}
-        uses: shivammathur/setup-php@2.24.0
+        uses: shivammathur/setup-php@2.25.5
         with:
           php-version: ${{ matrix.php-versions }}
           extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_sqlite, mysql, pdo_mysql, pgsql, pdo_pgsql, apcu
@@ -98,7 +98,7 @@ jobs:
           cat config/config.php
           ./occ user:list
           ./occ app:enable --force ${{ env.APP_NAME }}
-          ./occ config:system:set query_log_file --value '/home/runner/work/${{ env.APP_NAME }}/${{ env.APP_NAME }}/query.log'
+          ./occ config:system:set query_log_file --value "$PWD/query.log"
           php -S localhost:8080 &
 
       - name: Run behat
@@ -123,12 +123,12 @@ jobs:
                 myError += data.toString()
               }
             }
-            await exec.exec(`/bin/bash -c "cat /home/runner/work/${{ env.APP_NAME }}/${{ env.APP_NAME }}/query.log | wc -l"`, [], options)
+            await exec.exec(`/bin/bash -c "cat query.log | wc -l"`, [], options)
             msg = myOutput
             const queryCount = parseInt(myOutput, 10)
 
             myOutput = ''
-            await exec.exec('cat', ['/home/runner/work/${{ env.APP_NAME }}/${{ env.APP_NAME }}/apps/${{ env.APP_NAME }}/tests/integration/base-query-count.txt'], options)
+            await exec.exec('cat', ['apps/${{ env.APP_NAME }}/tests/integration/base-query-count.txt'], options)
             const baseCount = parseInt(myOutput, 10)
 
             const absoluteIncrease = queryCount - baseCount

cypress/support/e2e.js

@@ -16,5 +16,9 @@
 // Import commands.js using ES2015 syntax:
 import './commands.js'
 
+Cypress.on('uncaught:exception', (err) => {
+	return !err.message.includes('ResizeObserver loop limit exceeded')
+})
+
 // Alternatively you can use CommonJS syntax:
 // require('./commands')

docs/export-import.md

@@ -0,0 +1,98 @@
+## Export
+
+Deck currently supports exporting all boards a user owns in a single JSON file. The format is based on the database schema that deck uses. It can be used to re-import boards on the same or other instances.
+
+The export currently has some kown limitations in terms of specific data not included:
+- Activity information
+- File attachments to deck cards
+- Comments
+-
+```
+occ deck:export > my-file.json
+```
+
+## Import boards
+
+Importing can be done using the API or the `occ` `deck:import` command.
+
+It is possible to import from the following sources:
+
+### Deck JSON
+
+A json file that has been obtained from the above described `occ deck:export [userid]` command can be imported.
+
+```
+occ deck:import my-file.json
+```
+
+In case you are importing from a different instance you may use an additional config file to provide custom user id mapping in case users have different identifiers.
+
+```
+{
+    "owner": "admin",
+    "uidRelation": {
+        "johndoe": "test-user-1"
+    }
+}
+```
+
+#### Trello JSON
+
+Limitations:
+* Comments with more than 1000 characters are placed as attached files to the card.
+
+Steps:
+* Create the data file
+	* Access Trello
+	* go to the board you want to export
+	* Follow the steps in [Trello documentation](https://help.trello.com/article/747-exporting-data-from-trello-1) and export as JSON
+* Create the configuration file
+* Execute the import informing the import file path, data file and source as `Trello JSON`
+
+Create the configuration file respecting the [JSON Schema](https://github.com/nextcloud/deck/blob/main/lib/Service/Importer/fixtures/config-trelloJson-schema.json) for import `Trello JSON`
+
+Example configuration file:
+```json
+{
+    "owner": "admin",
+    "color": "0800fd",
+    "uidRelation": {
+        "johndoe": "johndoe"
+    }
+}
+```
+
+**Limitations**:
+
+Importing from a JSON file imports up to 1000 actions. To find out how many actions the board to be imported has, identify how many actions the JSON has.
+
+#### Trello API
+
+Import using API is recommended for boards with more than 1000 actions.
+
+Trello makes it possible to attach links to a card. Deck does not have this feature. Attachments and attachment links are added in a markdown table at the end of the description for every imported card that has attachments in Trello.
+
+* Get the API Key and API Token [here](https://developer.atlassian.com/cloud/trello/guides/rest-api/api-introduction/#authentication-and-authorization)
+* Get the ID of the board you want to import by making a request to:
+  https://api.trello.com/1/members/me/boards?key={yourKey}&token={yourToken}&fields=id,name
+
+  This ID you will use in the configuration file in the `board` property
+* Create the configuration file
+
+Create the configuration file respecting the [JSON Schema](https://github.com/nextcloud/deck/blob/main/lib/Service/Importer/fixtures/config-trelloApi-schema.json) for import `Trello JSON`
+
+Example configuration file:
+```json
+{
+    "owner": "admin",
+    "color": "0800fd",
+    "api": {
+        "key": "0cc175b9c0f1b6a831c399e269772661",
+        "token": "92eb5ffee6ae2fec3ad71c777531578f4a8a08f09d37b73795649038408b5f33"
+    },
+    "board": "8277e0910d750195b4487976",
+    "uidRelation": {
+        "johndoe": "johndoe"
+    }
+}
+```

lib/Command/BoardImport.php

@@ -25,17 +25,15 @@
 
 use OCA\Deck\Service\Importer\BoardImportCommandService;
 use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class BoardImport extends Command {
-	private BoardImportCommandService $boardImportCommandService;
-
 	public function __construct(
-		BoardImportCommandService $boardImportCommandService
+		private BoardImportCommandService $boardImportCommandService
 	) {
-		$this->boardImportCommandService = $boardImportCommandService;
 		parent::__construct();
 	}
 
@@ -44,7 +42,9 @@ public function __construct(
 	 */
 	protected function configure() {
 		$allowedSystems = $this->boardImportCommandService->getAllowedImportSystems();
-		$names = array_column($allowedSystems, 'name');
+		$names = array_map(function ($name) {
+			return '"' . $name . '"';
+		}, array_column($allowedSystems, 'internalName'));
 		$this
 			->setName('deck:import')
 			->setDescription('Import data')
@@ -53,7 +53,7 @@ protected function configure() {
 				null,
 				InputOption::VALUE_REQUIRED,
 				'Source system for import. Available options: ' . implode(', ', $names) . '.',
-				null
+				'DeckJson',
 			)
 			->addOption(
 				'config',
@@ -69,6 +69,11 @@ protected function configure() {
 				'Data file to import.',
 				'data.json'
 			)
+			->addArgument(
+				'file',
+				InputArgument::OPTIONAL,
+				'File to import',
+			)
 		;
 	}
 

lib/Command/UserExport.php

@@ -29,39 +29,23 @@
 use OCA\Deck\Db\StackMapper;
 use OCA\Deck\Model\CardDetails;
 use OCA\Deck\Service\BoardService;
-use OCP\AppFramework\Db\DoesNotExistException;
-use OCP\AppFramework\Db\MultipleObjectsReturnedException;
-use OCP\IGroupManager;
-use OCP\IUserManager;
+use OCP\App\IAppManager;
+use OCP\DB\Exception;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class UserExport extends Command {
-	protected $boardService;
-	protected $cardMapper;
-	private $userManager;
-	private $groupManager;
-	private $assignedUsersMapper;
-
-	public function __construct(BoardMapper $boardMapper,
-								BoardService $boardService,
-								StackMapper $stackMapper,
-								CardMapper $cardMapper,
-								AssignmentMapper $assignedUsersMapper,
-								IUserManager $userManager,
-								IGroupManager $groupManager) {
+	public function __construct(
+		private IAppManager $appManager,
+		private BoardMapper $boardMapper,
+		private BoardService $boardService,
+		private StackMapper $stackMapper,
+		private CardMapper $cardMapper,
+		private AssignmentMapper $assignedUsersMapper,
+	) {
 		parent::__construct();
-
-		$this->cardMapper = $cardMapper;
-		$this->boardService = $boardService;
-		$this->stackMapper = $stackMapper;
-		$this->assignedUsersMapper = $assignedUsersMapper;
-		$this->boardMapper = $boardMapper;
-
-		$this->userManager = $userManager;
-		$this->groupManager = $groupManager;
 	}
 
 	protected function configure() {
@@ -73,30 +57,27 @@ protected function configure() {
 				InputArgument::REQUIRED,
 				'User ID of the user'
 			)
+			->addOption('legacy-format', 'l')
 		;
 	}
 
 	/**
-	 * @param InputInterface $input
-	 * @param OutputInterface $output
-	 * @return int
-	 * @throws DoesNotExistException
-	 * @throws MultipleObjectsReturnedException
-	 * @throws \ReflectionException
+	 * @throws Exception
 	 */
 	protected function execute(InputInterface $input, OutputInterface $output): int {
 		$userId = $input->getArgument('user-id');
+		$legacyFormat = $input->getOption('legacy-format');
 
 		$this->boardService->setUserId($userId);
-		$boards = $this->boardService->findAll();
+		$boards = $this->boardService->findAll(fullDetails: false);
 
 		$data = [];
 		foreach ($boards as $board) {
 			$fullBoard = $this->boardMapper->find($board->getId(), true, true);
-			$data[$board->getId()] = (array)$fullBoard->jsonSerialize();
+			$data[$board->getId()] = $fullBoard->jsonSerialize();
 			$stacks = $this->stackMapper->findAll($board->getId());
 			foreach ($stacks as $stack) {
-				$data[$board->getId()]['stacks'][] = (array)$stack->jsonSerialize();
+				$data[$board->getId()]['stacks'][$stack->getId()] = $stack->jsonSerialize();
 				$cards = $this->cardMapper->findAllByStack($stack->getId());
 				foreach ($cards as $card) {
 					$fullCard = $this->cardMapper->find($card->getId());
@@ -108,7 +89,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 				}
 			}
 		}
-		$output->writeln(json_encode($data, JSON_PRETTY_PRINT));
+		$output->writeln(json_encode(
+			$legacyFormat ? $data : [
+				'version' => $this->appManager->getAppVersion('deck'),
+				'boards' => $data
+			],
+			JSON_PRETTY_PRINT));
 		return 0;
 	}
 }

lib/Db/Assignment.php

@@ -41,4 +41,17 @@ public function __construct() {
 		$this->addType('type', 'integer');
 		$this->addResolvable('participant');
 	}
+
+	public function getTypeString(): string {
+		switch ($this->getType()) {
+			case self::TYPE_USER:
+				return 'user';
+			case self::TYPE_GROUP:
+				return 'group';
+			case self::TYPE_CIRCLE:
+				return 'circle';
+		}
+
+		return 'unknown';
+	}
 }

lib/Db/BoardMapper.php

@@ -532,12 +532,12 @@ public function flushCache(?int $boardId = null, ?string $userId = null) {
 		if ($boardId) {
 			unset($this->boardCache[$boardId]);
 		} else {
-			$this->boardCache = null;
+			$this->boardCache = new CappedMemoryCache();
 		}
 		if ($userId) {
 			unset($this->userBoardCache[$userId]);
 		} else {
-			$this->userBoardCache = null;
+			$this->userBoardCache = new CappedMemoryCache();
 		}
 	}
 }

lib/Db/LabelMapper.php

@@ -115,7 +115,9 @@ public function findAssignedLabelsForBoard($boardId, $limit = null, $offset = nu
 	}
 
 	public function insert(Entity $entity): Entity {
-		$entity->setLastModified(time());
+		if (!in_array('lastModified', $entity->getUpdatedFields())) {
+			$entity->setLastModified(time());
+		}
 		return parent::insert($entity);
 	}
 

lib/Db/RelationalEntity.php

@@ -138,7 +138,7 @@ public function __call(string $methodName, array $args) {
 
 		$attr = lcfirst(substr($methodName, 3));
 		if (array_key_exists($attr, $this->_resolvedProperties) && strpos($methodName, 'set') === 0) {
-			if (!is_scalar($args[0])) {
+			if ($args[0] !== null && !is_scalar($args[0])) {
 				$args[0] = $args[0]['primaryKey'];
 			}
 			parent::setter($attr, $args);

lib/Service/Importer/ABoardImportService.php

@@ -61,6 +61,10 @@ abstract class ABoardImportService {
 	 */
 	abstract public function bootstrap(): void;
 
+	public function getBoards(): array {
+		return [$this->getImportService()->getData()];
+	}
+
 	abstract public function getBoard(): ?Board;
 
 	/**
@@ -133,4 +137,13 @@ public function getImportService(): BoardImportService {
 	public function needValidateData(): bool {
 		return $this->needValidateData;
 	}
+
+	public function reset(): void {
+		// FIXME: Would be cleaner if we could just get a new instance per board
+		// but currently https://github.com/nextcloud/deck/blob/7d820aa3f9fc69ada8188549b9a2fbb9093ffb95/lib/Service/Importer/BoardImportService.php#L194 returns a singleton
+		$this->labels = [];
+		$this->stacks = [];
+		$this->acls = [];
+		$this->cards = [];
+	}
 }