From 72b5e1baf66c3c9794aaba70e51a7413b0d211ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Mon, 17 Jul 2023 16:54:20 +0200 Subject: [PATCH 01/26] fix: Properly export cards as a child element of the related stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Command/UserExport.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Command/UserExport.php b/lib/Command/UserExport.php index 3b2948270..bdee865ea 100644 --- a/lib/Command/UserExport.php +++ b/lib/Command/UserExport.php @@ -96,7 +96,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $data[$board->getId()] = (array)$fullBoard->jsonSerialize(); $stacks = $this->stackMapper->findAll($board->getId()); foreach ($stacks as $stack) { - $data[$board->getId()]['stacks'][] = (array)$stack->jsonSerialize(); + $data[$board->getId()]['stacks'][$stack->getId()] = (array)$stack->jsonSerialize(); $cards = $this->cardMapper->findAllByStack($stack->getId()); foreach ($cards as $card) { $fullCard = $this->cardMapper->find($card->getId()); From ef463f32e66629ac8c5b334df7d02676692a11cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Mon, 17 Jul 2023 16:54:41 +0200 Subject: [PATCH 02/26] fix: Avoid failing due to uninitialized acces of systemInstance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Service/Importer/BoardImportService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/Importer/BoardImportService.php b/lib/Service/Importer/BoardImportService.php index 657f938b2..ee1ff7dc7 100644 --- a/lib/Service/Importer/BoardImportService.php +++ b/lib/Service/Importer/BoardImportService.php @@ -61,7 +61,7 @@ class BoardImportService { private ICommentsManager $commentsManager; private IEventDispatcher $eventDispatcher; private string $system = ''; - private ?ABoardImportService $systemInstance; + private ?ABoardImportService $systemInstance = null; private array $allowedSystems = []; /** * Data object created from config JSON From 8954569aee075ec51cc5c332ed846e36dfd66440 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 12 Jul 2023 13:03:17 +0200 Subject: [PATCH 03/26] WIP: enh(import): import deck json exports Signed-off-by: Max --- .../Importer/BoardImportCommandService.php | 4 +- lib/Service/Importer/BoardImportService.php | 6 + .../Importer/Systems/DeckJsonService.php | 241 ++++++++++++++++++ .../fixtures/config-deckJson-schema.json | 24 ++ 4 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 lib/Service/Importer/Systems/DeckJsonService.php create mode 100644 lib/Service/Importer/fixtures/config-deckJson-schema.json diff --git a/lib/Service/Importer/BoardImportCommandService.php b/lib/Service/Importer/BoardImportCommandService.php index d45c784de..ced94fd47 100644 --- a/lib/Service/Importer/BoardImportCommandService.php +++ b/lib/Service/Importer/BoardImportCommandService.php @@ -95,7 +95,7 @@ protected function validateConfig(): void { $helper = $this->getCommand()->getHelper('question'); $question = new Question( "You can get more info on https://deck.readthedocs.io/en/latest/User_documentation_en/#6-import-boards\n" . - 'Please inform a valid config json file: ', + 'Please provide a valid config json file: ', 'config.json' ); $question->setValidator(function (string $answer) { @@ -130,7 +130,7 @@ public function validateSystem(): void { $allowedSystems = $this->getAllowedImportSystems(); $names = array_column($allowedSystems, 'name'); $question = new ChoiceQuestion( - 'Please inform a source system', + 'Please select a source system', $names, 0 ); diff --git a/lib/Service/Importer/BoardImportService.php b/lib/Service/Importer/BoardImportService.php index ee1ff7dc7..20fd959f6 100644 --- a/lib/Service/Importer/BoardImportService.php +++ b/lib/Service/Importer/BoardImportService.php @@ -42,6 +42,7 @@ use OCA\Deck\Service\FileService; use OCA\Deck\Service\Importer\Systems\TrelloApiService; use OCA\Deck\Service\Importer\Systems\TrelloJsonService; +use OCA\Deck\Service\Importer\Systems\DeckJsonService; use OCP\Comments\IComment; use OCP\Comments\ICommentsManager; use OCP\Comments\NotFoundException as CommentNotFoundException; @@ -174,6 +175,11 @@ public function getAllowedImportSystems(): array { 'class' => TrelloJsonService::class, 'internalName' => 'TrelloJson' ]); + $this->addAllowedImportSystem([ + 'name' => DeckJsonService::$name, + 'class' => DeckJsonService::class, + 'internalName' => 'DeckJson' + ]); } $this->eventDispatcher->dispatchTyped(new BoardImportGetAllowedEvent($this)); return $this->allowedSystems; diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php new file mode 100644 index 000000000..70b1f23a2 --- /dev/null +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -0,0 +1,241 @@ + + * + * @author Vitor Mattos + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Deck\Service\Importer\Systems; + +use OC\Comments\Comment; +use OCA\Deck\BadRequestException; +use OCA\Deck\Db\Acl; +use OCA\Deck\Db\Assignment; +use OCA\Deck\Db\Attachment; +use OCA\Deck\Db\Board; +use OCA\Deck\Db\Card; +use OCA\Deck\Db\Label; +use OCA\Deck\Db\Stack; +use OCA\Deck\Service\Importer\ABoardImportService; +use OCP\Comments\IComment; +use OCP\IL10N; +use OCP\IURLGenerator; +use OCP\IUser; +use OCP\IUserManager; + +class DeckJsonService extends ABoardImportService { + /** @var string */ + public static $name = 'Deck JSON'; + /** @var IUserManager */ + private $userManager; + /** @var IURLGenerator */ + private $urlGenerator; + /** @var IL10N */ + private $l10n; + /** @var IUser[] */ + private $members = []; + private array $tmpCards = []; + + public function __construct( + IUserManager $userManager, + IURLGenerator $urlGenerator, + IL10N $l10n + ) { + $this->userManager = $userManager; + $this->urlGenerator = $urlGenerator; + $this->l10n = $l10n; + } + + public function bootstrap(): void { + $this->validateUsers(); + } + + public function getJsonSchemaPath(): string { + return implode(DIRECTORY_SEPARATOR, [ + __DIR__, + '..', + 'fixtures', + 'config-deckJson-schema.json', + ]); + } + + public function validateUsers(): void { + if (empty($this->getImportService()->getConfig('uidRelation'))) { + return; + } + foreach ($this->getImportService()->getConfig('uidRelation') as $exportUid => $nextcloudUid) { + $user = array_filter($this->getImportService()->getData()->members, function (\stdClass $u) use ($exportUid) { + return $u->username === $exportUid; + }); + if (!$user) { + throw new \LogicException('Trello user ' . $exportUid . ' not found in property "members" of json data'); + } + if (!is_string($nextcloudUid) && !is_numeric($nextcloudUid)) { + throw new \LogicException('User on setting uidRelation is invalid'); + } + $nextcloudUid = (string) $nextcloudUid; + $this->getImportService()->getConfig('uidRelation')->$exportUid = $this->userManager->get($nextcloudUid); + if (!$this->getImportService()->getConfig('uidRelation')->$exportUid) { + throw new \LogicException('User on setting uidRelation not found: ' . $nextcloudUid); + } + $user = current($user); + $this->members[$user->id] = $this->getImportService()->getConfig('uidRelation')->$exportUid; + } + } + + public function getCardAssignments(): array { + $assignments = []; + foreach ($this->tmpCards as $sourceCard) { + foreach ($sourceCard->assignedUsers as $idMember) { + $assignment = new Assignment(); + $assignment->setCardId($this->cards[$sourceCard->id]->getId()); + $assignment->setParticipant($idMember->participant->uid); + $assignment->setType($idMember->participant->type); + $assignments[$sourceCard->id][] = $assignment; + } + } + return $assignments; + } + + public function getComments(): array { + // Comments are not implemented in export + return []; + } + + public function getCardLabelAssignment(): array { + $cardsLabels = []; + foreach ($this->tmpCards as $sourceCard) { + foreach ($sourceCard->labels as $label) { + $cardId = $this->cards[$sourceCard->id]->getId(); + $labelId = $this->labels[$label->id]->getId(); + $cardsLabels[$cardId][] = $labelId; + } + } + return $cardsLabels; + } + + public function getBoard(): Board { + $board = $this->getImportService()->getBoard(); + if (empty($this->getImportService()->getData()->title)) { + throw new BadRequestException('Invalid name of board'); + } + $board->setTitle($this->getImportService()->getData()->title); + $board->setOwner($this->getImportService()->getData()->owner->uid); + $board->setColor($this->getImportService()->getData()->color); + return $board; + } + + /** + * @return Label[] + */ + public function getLabels(): array { + foreach ($this->getImportService()->getData()->labels as $label) { + $newLabel = new Label(); + $newLabel->setTitle($label->title); + $newLabel->setColor($label->color); + $newLabel->setBoardId($this->getImportService()->getBoard()->getId()); + $this->labels[$label->id] = $newLabel; + } + return $this->labels; + } + + /** + * @return Stack[] + */ + public function getStacks(): array { + $return = []; + foreach ($this->getImportService()->getData()->stacks as $index => $source) { + if ($source->title) { + $stack = new Stack(); + $stack->setTitle($source->title); + $stack->setBoardId($this->getImportService()->getBoard()->getId()); + $stack->setOrder($source->order); + $return[$source->id] = $stack; + } + + if ($source->cards) { + foreach ($source->cards as $card) { + $card->stackId = $index; + $this->tmpCards[] = $card; + } + // TODO: check older exports as currently there is a bug that adds lists to it with different index + } + } + return $return; + } + + /** + * @return Card[] + */ + public function getCards(): array { + $cards = []; + foreach ($this->tmpCards as $cardSource) { + $card = new Card(); + $card->setTitle($cardSource->title); + $card->setLastModified($cardSource->lastModified); + $card->setArchived($cardSource->archived); + $card->setDescription($cardSource->description); + $card->setStackId($this->stacks[$cardSource->stackId]->getId()); + $card->setType('plain'); + $card->setOrder($cardSource->order); + $card->setOwner($this->getBoard()->getOwner()); + $card->setDuedate($cardSource->duedate); + $cards[$cardSource->id] = $card; + } + return $cards; + } + + /** + * @return Acl[] + */ + public function getAclList(): array { + // FIXME: To implement + $return = []; + foreach ($this->members as $member) { + if ($member->getUID() === $this->getImportService()->getConfig('owner')->getUID()) { + continue; + } + $acl = new Acl(); + $acl->setBoardId($this->getImportService()->getBoard()->getId()); + $acl->setType(Acl::PERMISSION_TYPE_USER); + $acl->setParticipant($member->getUID()); + $acl->setPermissionEdit(false); + $acl->setPermissionShare(false); + $acl->setPermissionManage(false); + $return[] = $acl; + } + return $return; + } + + private function replaceUsernames(string $text): string { + foreach ($this->getImportService()->getConfig('uidRelation') as $trello => $nextcloud) { + $text = str_replace($trello, $nextcloud->getUID(), $text); + } + return $text; + } + + public function getBoards(): array { + return get_object_vars($this->getImportService()->getData()); + } + + public function reset(): void { + parent::reset(); + $this->tmpCards = []; + } +} diff --git a/lib/Service/Importer/fixtures/config-deckJson-schema.json b/lib/Service/Importer/fixtures/config-deckJson-schema.json new file mode 100644 index 000000000..7635727c1 --- /dev/null +++ b/lib/Service/Importer/fixtures/config-deckJson-schema.json @@ -0,0 +1,24 @@ +{ + "type": "object", + "properties": { + "uidRelation": { + "type": "object", + "comment": "Relationship between Trello and Nextcloud usernames", + "example": { + "johndoe": "admin" + } + }, + "owner": { + "type": "string", + "required": true, + "comment": "Nextcloud owner username" + }, + "color": { + "type": "string", + "required": true, + "pattern": "^[0-9a-fA-F]{6}$", + "comment": "Default color for the board. If you don't inform, the default color will be used.", + "default": "0800fd" + } + } +} \ No newline at end of file From 2ae9f7148393f35d7aca505bc40d28feb81eb072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Mon, 17 Jul 2023 16:52:00 +0200 Subject: [PATCH 04/26] feat: Implement logic to import multiple boards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Command/BoardImport.php | 5 +- lib/Service/Importer/ABoardImportService.php | 13 ++++ .../Importer/BoardImportCommandService.php | 39 ++++++---- lib/Service/Importer/BoardImportService.php | 77 ++++++++----------- .../Importer/Systems/DeckJsonService.php | 29 +++---- .../Importer/Systems/TrelloJsonService.php | 8 ++ 6 files changed, 88 insertions(+), 83 deletions(-) diff --git a/lib/Command/BoardImport.php b/lib/Command/BoardImport.php index ae0388f91..4c70c61ae 100644 --- a/lib/Command/BoardImport.php +++ b/lib/Command/BoardImport.php @@ -30,12 +30,9 @@ use Symfony\Component\Console\Output\OutputInterface; class BoardImport extends Command { - private BoardImportCommandService $boardImportCommandService; - public function __construct( - BoardImportCommandService $boardImportCommandService + private BoardImportCommandService $boardImportCommandService ) { - $this->boardImportCommandService = $boardImportCommandService; parent::__construct(); } diff --git a/lib/Service/Importer/ABoardImportService.php b/lib/Service/Importer/ABoardImportService.php index 2e6acb605..43ec745f5 100644 --- a/lib/Service/Importer/ABoardImportService.php +++ b/lib/Service/Importer/ABoardImportService.php @@ -61,6 +61,10 @@ abstract class ABoardImportService { */ abstract public function bootstrap(): void; + public function getBoards(): array { + return [$this->getImportService()->getData()]; + } + abstract public function getBoard(): ?Board; /** @@ -133,4 +137,13 @@ public function getImportService(): BoardImportService { public function needValidateData(): bool { return $this->needValidateData; } + + public function reset(): void { + // FIXME: Would be cleaner if we could just get a new instance per board + // but currently https://github.com/nextcloud/deck/blob/7d820aa3f9fc69ada8188549b9a2fbb9093ffb95/lib/Service/Importer/BoardImportService.php#L194 returns a singleton + $this->labels = []; + $this->stacks = []; + $this->acls = []; + $this->cards = []; + } } diff --git a/lib/Service/Importer/BoardImportCommandService.php b/lib/Service/Importer/BoardImportCommandService.php index ced94fd47..0aea5ca97 100644 --- a/lib/Service/Importer/BoardImportCommandService.php +++ b/lib/Service/Importer/BoardImportCommandService.php @@ -179,21 +179,28 @@ public function bootstrap(): void { public function import(): void { $this->getOutput()->writeln('Starting import...'); $this->bootstrap(); - $this->getOutput()->writeln('Importing board...'); - $this->importBoard(); - $this->getOutput()->writeln('Assign users to board...'); - $this->importAcl(); - $this->getOutput()->writeln('Importing labels...'); - $this->importLabels(); - $this->getOutput()->writeln('Importing stacks...'); - $this->importStacks(); - $this->getOutput()->writeln('Importing cards...'); - $this->importCards(); - $this->getOutput()->writeln('Assign cards to labels...'); - $this->assignCardsToLabels(); - $this->getOutput()->writeln('Importing comments...'); - $this->importComments(); - $this->getOutput()->writeln('Importing participants...'); - $this->importCardAssignments(); + $boards = $this->getImportSystem()->getBoards(); + + foreach ($boards as $board) { + $this->reset(); + $this->setData($board); + $this->getOutput()->writeln('Importing board...'); + $this->importBoard(); + $this->getOutput()->writeln('Assign users to board...'); + $this->importAcl(); + $this->getOutput()->writeln('Importing labels...'); + $this->importLabels(); + $this->getOutput()->writeln('Importing stacks...'); + $this->importStacks(); + $this->getOutput()->writeln('Importing cards...'); + $this->importCards(); + $this->getOutput()->writeln('Assign cards to labels...'); + $this->assignCardsToLabels(); + $this->getOutput()->writeln('Importing comments...'); + $this->importComments(); + $this->getOutput()->writeln('Importing participants...'); + $this->importCardAssignments(); + $this->getOutput()->writeln('Finished board import of "' . $this->getBoard()->getTitle() . '"'); + } } } diff --git a/lib/Service/Importer/BoardImportService.php b/lib/Service/Importer/BoardImportService.php index 20fd959f6..364fd1bce 100644 --- a/lib/Service/Importer/BoardImportService.php +++ b/lib/Service/Importer/BoardImportService.php @@ -40,9 +40,9 @@ use OCA\Deck\Exceptions\ConflictException; use OCA\Deck\NotFoundException; use OCA\Deck\Service\FileService; +use OCA\Deck\Service\Importer\Systems\DeckJsonService; use OCA\Deck\Service\Importer\Systems\TrelloApiService; use OCA\Deck\Service\Importer\Systems\TrelloJsonService; -use OCA\Deck\Service\Importer\Systems\DeckJsonService; use OCP\Comments\IComment; use OCP\Comments\ICommentsManager; use OCP\Comments\NotFoundException as CommentNotFoundException; @@ -51,16 +51,6 @@ use OCP\Server; class BoardImportService { - private IUserManager $userManager; - private BoardMapper $boardMapper; - private AclMapper $aclMapper; - private LabelMapper $labelMapper; - private StackMapper $stackMapper; - private CardMapper $cardMapper; - private AssignmentMapper $assignmentMapper; - private AttachmentMapper $attachmentMapper; - private ICommentsManager $commentsManager; - private IEventDispatcher $eventDispatcher; private string $system = ''; private ?ABoardImportService $systemInstance = null; private array $allowedSystems = []; @@ -81,27 +71,17 @@ class BoardImportService { private Board $board; public function __construct( - IUserManager $userManager, - BoardMapper $boardMapper, - AclMapper $aclMapper, - LabelMapper $labelMapper, - StackMapper $stackMapper, - AssignmentMapper $assignmentMapper, - AttachmentMapper $attachmentMapper, - CardMapper $cardMapper, - ICommentsManager $commentsManager, - IEventDispatcher $eventDispatcher + private IUserManager $userManager, + private BoardMapper $boardMapper, + private AclMapper $aclMapper, + private LabelMapper $labelMapper, + private StackMapper $stackMapper, + private AssignmentMapper $assignmentMapper, + private AttachmentMapper $attachmentMapper, + private CardMapper $cardMapper, + private ICommentsManager $commentsManager, + private IEventDispatcher $eventDispatcher ) { - $this->userManager = $userManager; - $this->boardMapper = $boardMapper; - $this->aclMapper = $aclMapper; - $this->labelMapper = $labelMapper; - $this->stackMapper = $stackMapper; - $this->cardMapper = $cardMapper; - $this->assignmentMapper = $assignmentMapper; - $this->attachmentMapper = $attachmentMapper; - $this->commentsManager = $commentsManager; - $this->eventDispatcher = $eventDispatcher; $this->board = new Board(); $this->disableCommentsEvents(); } @@ -121,17 +101,23 @@ private function disableCommentsEvents(): void { public function import(): void { $this->bootstrap(); - try { - $this->importBoard(); - $this->importAcl(); - $this->importLabels(); - $this->importStacks(); - $this->importCards(); - $this->assignCardsToLabels(); - $this->importComments(); - $this->importCardAssignments(); - } catch (\Throwable $th) { - throw new BadRequestException($th->getMessage()); + $boards = $this->getImportSystem()->getBoards(); + foreach ($boards as $board) { + try { + $this->reset(); + $this->setData($board); + $this->importBoard(); + $this->importAcl(); + $this->importLabels(); + $this->importStacks(); + $this->importCards(); + $this->assignCardsToLabels(); + $this->importComments(); + $this->importCardAssignments(); + } catch (\Throwable $th) { + throw $th; + throw new BadRequestException($th->getMessage()); + } } } @@ -139,7 +125,7 @@ public function validateSystem(): void { $allowedSystems = $this->getAllowedImportSystems(); $allowedSystems = array_column($allowedSystems, 'internalName'); if (!in_array($this->getSystem(), $allowedSystems)) { - throw new NotFoundException('Invalid system'); + throw new NotFoundException('Invalid system: ' . $this->getSystem()); } } @@ -201,6 +187,11 @@ public function setImportSystem(ABoardImportService $instance): void { $this->systemInstance = $instance; } + public function reset(): void { + $this->board = new Board(); + $this->getImportSystem()->reset(); + } + public function importBoard(): void { $board = $this->getImportSystem()->getBoard(); if ($board) { diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index 70b1f23a2..03d3c2e6e 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -23,43 +23,30 @@ namespace OCA\Deck\Service\Importer\Systems; -use OC\Comments\Comment; use OCA\Deck\BadRequestException; use OCA\Deck\Db\Acl; use OCA\Deck\Db\Assignment; -use OCA\Deck\Db\Attachment; use OCA\Deck\Db\Board; use OCA\Deck\Db\Card; use OCA\Deck\Db\Label; use OCA\Deck\Db\Stack; use OCA\Deck\Service\Importer\ABoardImportService; -use OCP\Comments\IComment; use OCP\IL10N; use OCP\IURLGenerator; use OCP\IUser; use OCP\IUserManager; class DeckJsonService extends ABoardImportService { - /** @var string */ public static $name = 'Deck JSON'; - /** @var IUserManager */ - private $userManager; - /** @var IURLGenerator */ - private $urlGenerator; - /** @var IL10N */ - private $l10n; /** @var IUser[] */ - private $members = []; + private array $members = []; private array $tmpCards = []; public function __construct( - IUserManager $userManager, - IURLGenerator $urlGenerator, - IL10N $l10n + private IUserManager $userManager, + private IURLGenerator $urlGenerator, + private IL10N $l10n ) { - $this->userManager = $userManager; - $this->urlGenerator = $urlGenerator; - $this->l10n = $l10n; } public function bootstrap(): void { @@ -76,7 +63,7 @@ public function getJsonSchemaPath(): string { } public function validateUsers(): void { - if (empty($this->getImportService()->getConfig('uidRelation'))) { + if (empty($this->getImportService()->getConfig('uidRelation')) || !isset($this->getImportService()->getData()->members)) { return; } foreach ($this->getImportService()->getConfig('uidRelation') as $exportUid => $nextcloudUid) { @@ -169,7 +156,7 @@ public function getStacks(): array { $return[$source->id] = $stack; } - if ($source->cards) { + if (isset($source->cards)) { foreach ($source->cards as $card) { $card->stackId = $index; $this->tmpCards[] = $card; @@ -231,7 +218,9 @@ private function replaceUsernames(string $text): string { } public function getBoards(): array { - return get_object_vars($this->getImportService()->getData()); + // Old format has just the raw board data, new one a key boards + $data = $this->getImportService()->getData(); + return array_values((array)($data->boards ?? $data)); } public function reset(): void { diff --git a/lib/Service/Importer/Systems/TrelloJsonService.php b/lib/Service/Importer/Systems/TrelloJsonService.php index 88c4ae23a..a4a3673c1 100644 --- a/lib/Service/Importer/Systems/TrelloJsonService.php +++ b/lib/Service/Importer/Systems/TrelloJsonService.php @@ -397,4 +397,12 @@ private function appendAttachmentsToDescription(\stdClass $trelloCard): void { $trelloCard->desc .= "| [{$name}]({$attachment->url}) | {$attachment->date} |\n"; } } + + public function getBoards(): array { + if ($this->getImportService()->getData()->boards) { + return $this->getImportService()->getData()->boards; + } + + return [$this->getImportService()->getData()]; + } } From e98188fbf8a6bce0cff6c0bb1ce315136e182b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Mon, 17 Jul 2023 20:07:53 +0200 Subject: [PATCH 05/26] feat: Add app version to the deck app export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Command/UserExport.php | 56 ++++++++++++++------------------------ 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/lib/Command/UserExport.php b/lib/Command/UserExport.php index bdee865ea..4d869e275 100644 --- a/lib/Command/UserExport.php +++ b/lib/Command/UserExport.php @@ -29,39 +29,23 @@ use OCA\Deck\Db\StackMapper; use OCA\Deck\Model\CardDetails; use OCA\Deck\Service\BoardService; -use OCP\AppFramework\Db\DoesNotExistException; -use OCP\AppFramework\Db\MultipleObjectsReturnedException; -use OCP\IGroupManager; -use OCP\IUserManager; +use OCP\App\IAppManager; +use OCP\DB\Exception; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; class UserExport extends Command { - protected $boardService; - protected $cardMapper; - private $userManager; - private $groupManager; - private $assignedUsersMapper; - - public function __construct(BoardMapper $boardMapper, - BoardService $boardService, - StackMapper $stackMapper, - CardMapper $cardMapper, - AssignmentMapper $assignedUsersMapper, - IUserManager $userManager, - IGroupManager $groupManager) { + public function __construct( + private IAppManager $appManager, + private BoardMapper $boardMapper, + private BoardService $boardService, + private StackMapper $stackMapper, + private CardMapper $cardMapper, + private AssignmentMapper $assignedUsersMapper, + ) { parent::__construct(); - - $this->cardMapper = $cardMapper; - $this->boardService = $boardService; - $this->stackMapper = $stackMapper; - $this->assignedUsersMapper = $assignedUsersMapper; - $this->boardMapper = $boardMapper; - - $this->userManager = $userManager; - $this->groupManager = $groupManager; } protected function configure() { @@ -73,19 +57,16 @@ protected function configure() { InputArgument::REQUIRED, 'User ID of the user' ) + ->addOption('legacy-format', 'l') ; } /** - * @param InputInterface $input - * @param OutputInterface $output - * @return int - * @throws DoesNotExistException - * @throws MultipleObjectsReturnedException - * @throws \ReflectionException + * @throws Exception */ protected function execute(InputInterface $input, OutputInterface $output): int { $userId = $input->getArgument('user-id'); + $legacyFormat = $input->getOption('legacy-format'); $this->boardService->setUserId($userId); $boards = $this->boardService->findAll(); @@ -93,10 +74,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int $data = []; foreach ($boards as $board) { $fullBoard = $this->boardMapper->find($board->getId(), true, true); - $data[$board->getId()] = (array)$fullBoard->jsonSerialize(); + $data[$board->getId()] = $fullBoard->jsonSerialize(); $stacks = $this->stackMapper->findAll($board->getId()); foreach ($stacks as $stack) { - $data[$board->getId()]['stacks'][$stack->getId()] = (array)$stack->jsonSerialize(); + $data[$board->getId()]['stacks'][$stack->getId()] = $stack->jsonSerialize(); $cards = $this->cardMapper->findAllByStack($stack->getId()); foreach ($cards as $card) { $fullCard = $this->cardMapper->find($card->getId()); @@ -108,7 +89,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int } } } - $output->writeln(json_encode($data, JSON_PRETTY_PRINT)); + $output->writeln(json_encode( + $legacyFormat ? $data : [ + 'version' => $this->appManager->getAppVersion('deck'), + 'boards' => $data + ], + JSON_PRETTY_PRINT)); return 0; } } From d93d4af44f214ef2835e0d17ebce7a978b31f1d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Tue, 18 Jul 2023 10:56:25 +0200 Subject: [PATCH 06/26] test: Add some basic integration test skeleton for import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- tests/integration/import/ImportExportTest.php | 132 ++++++++++++++++++ tests/phpunit.integration.xml | 3 + tests/unit/Command/UserExportTest.php | 8 +- .../Importer/Systems/DeckJsonServiceTest.php | 86 ++++++++++++ 4 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 tests/integration/import/ImportExportTest.php create mode 100644 tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php diff --git a/tests/integration/import/ImportExportTest.php b/tests/integration/import/ImportExportTest.php new file mode 100644 index 000000000..f6e339d6a --- /dev/null +++ b/tests/integration/import/ImportExportTest.php @@ -0,0 +1,132 @@ + + * + * @author Julius Härtl + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Deck\Db; + +use OCA\Deck\Command\BoardImport; +use OCA\Deck\Service\Importer\BoardImportService; +use OCA\Deck\Service\Importer\Systems\DeckJsonService; +use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\IUserManager; +use OCP\Server; +use Symfony\Component\Console\Application; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +/** + * @group DB + */ +class ImportExportTest extends \Test\TestCase { + + private IDBConnection $connection; + private const TEST_USER1 = 'test-share-user1'; + private const TEST_USER3 = 'test-share-user3'; + private const TEST_USER2 = 'test-share-user2'; + private const TEST_USER4 = 'test-share-user4'; + private const TEST_GROUP1 = 'test-share-group1'; + + public static function setUpBeforeClass(): void { + parent::setUpBeforeClass(); + + $backend = new \Test\Util\User\Dummy(); + \OC_User::useBackend($backend); + Server::get(IUserManager::class)->registerBackend($backend); + $backend->createUser(self::TEST_USER1, self::TEST_USER1); + $backend->createUser(self::TEST_USER2, self::TEST_USER2); + $backend->createUser(self::TEST_USER3, self::TEST_USER3); + $backend->createUser(self::TEST_USER4, self::TEST_USER4); + // create group + $groupBackend = new \Test\Util\Group\Dummy(); + $groupBackend->createGroup(self::TEST_GROUP1); + $groupBackend->createGroup('group'); + $groupBackend->createGroup('group1'); + $groupBackend->createGroup('group2'); + $groupBackend->createGroup('group3'); + $groupBackend->addToGroup(self::TEST_USER1, 'group'); + $groupBackend->addToGroup(self::TEST_USER2, 'group'); + $groupBackend->addToGroup(self::TEST_USER3, 'group'); + $groupBackend->addToGroup(self::TEST_USER2, 'group1'); + $groupBackend->addToGroup(self::TEST_USER3, 'group2'); + $groupBackend->addToGroup(self::TEST_USER4, 'group3'); + $groupBackend->addToGroup(self::TEST_USER2, self::TEST_GROUP1); + Server::get(IGroupManager::class)->addBackend($groupBackend); + } + + public function setUp(): void { + parent::setUp(); + + $this->connection = \OCP\Server::get(IDBConnection::class); + $this->connection->beginTransaction(); + + } + + public function testImportOcc() { + $input = $this->createMock(InputInterface::class); + $input->expects($this->any()) + ->method('getOption') + ->willReturnCallback(function ($arg) { + return match ($arg) { + 'system' => 'DeckJson', + 'data' => __DIR__ . '/../../data/deck.json', + 'config' => __DIR__ . '/../../data/config-trelloJson.json', + }; + }); + $output = $this->createMock(OutputInterface::class); + $importer = \OCP\Server::get(BoardImport::class); + $application = new Application(); + $importer->setApplication($application); + $importer->run($input, $output); + + $this->assertDatabase(); + } + + public function testImport() { + $importer = \OCP\Server::get(BoardImportService::class); + $deckJsonService = \OCP\Server::get(DeckJsonService::class); + $deckJsonService->setImportService($importer); + + $importer->setSystem('DeckJson'); + $importer->setImportSystem($deckJsonService); + $importer->setConfigInstance(json_decode(file_get_contents(__DIR__ . '/../../data/config-trelloJson.json'))); + $importer->setData(json_decode(file_get_contents(__DIR__ . '/../../data/deck.json'))); + $importer->import(); + + $this->assertDatabase(); + } + + public function assertDatabase() { + $boardMapper = \OCP\Server::get(BoardMapper::class); + $boards = $boardMapper->findAllByOwner('admin'); + self::assertEquals('My test board', $boards[0]->getTitle()); + self::assertEquals('Shared board', $boards[1]->getTitle()); + self::assertEquals(2, count($boards)); + } + + public function tearDown(): void { + if ($this->connection->inTransaction()) { + $this->connection->rollBack(); + } + parent::tearDown(); + } +} diff --git a/tests/phpunit.integration.xml b/tests/phpunit.integration.xml index f62881f9a..e5534edc8 100644 --- a/tests/phpunit.integration.xml +++ b/tests/phpunit.integration.xml @@ -12,5 +12,8 @@ ./integration/app + + ./integration/import + diff --git a/tests/unit/Command/UserExportTest.php b/tests/unit/Command/UserExportTest.php index 81fc49642..1d75a35b9 100644 --- a/tests/unit/Command/UserExportTest.php +++ b/tests/unit/Command/UserExportTest.php @@ -31,12 +31,14 @@ use OCA\Deck\Db\Stack; use OCA\Deck\Db\StackMapper; use OCA\Deck\Service\BoardService; +use OCP\App\IAppManager; use OCP\IGroupManager; use OCP\IUserManager; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; class UserExportTest extends \Test\TestCase { + protected $appManager; protected $boardMapper; protected $boardService; protected $stackMapper; @@ -45,10 +47,11 @@ class UserExportTest extends \Test\TestCase { protected $userManager; protected $groupManager; - private $userExport; + private UserExport $userExport; public function setUp(): void { parent::setUp(); + $this->appManager = $this->createMock(IAppManager::class); $this->boardMapper = $this->createMock(BoardMapper::class); $this->boardService = $this->createMock(BoardService::class); $this->stackMapper = $this->createMock(StackMapper::class); @@ -56,7 +59,7 @@ public function setUp(): void { $this->assignedUserMapper = $this->createMock(AssignmentMapper::class); $this->userManager = $this->createMock(IUserManager::class); $this->groupManager = $this->createMock(IGroupManager::class); - $this->userExport = new UserExport($this->boardMapper, $this->boardService, $this->stackMapper, $this->cardMapper, $this->assignedUserMapper, $this->userManager, $this->groupManager); + $this->userExport = new UserExport($this->appManager, $this->boardMapper, $this->boardService, $this->stackMapper, $this->cardMapper, $this->assignedUserMapper, $this->userManager, $this->groupManager); } public function getBoard($id) { @@ -114,5 +117,6 @@ public function testExecute() { ->method('findAll') ->willReturn([]); $result = $this->invokePrivate($this->userExport, 'execute', [$input, $output]); + self::assertEquals(0, $result); } } diff --git a/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php b/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php new file mode 100644 index 000000000..1a2c41aa1 --- /dev/null +++ b/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php @@ -0,0 +1,86 @@ + + * + * @author Vitor Mattos + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\Deck\Service\Importer\Systems; + +use OCA\Deck\Service\Importer\BoardImportService; +use OCP\IL10N; +use OCP\IURLGenerator; +use OCP\IUser; +use OCP\IUserManager; +use OCP\Server; +use PHPUnit\Framework\MockObject\MockObject; + +/** + * @group DB + */ +class DeckJsonServiceTest extends \Test\TestCase { + private DeckJsonService $service; + /** @var IURLGenerator|MockObject */ + private $urlGenerator; + /** @var IUserManager|MockObject */ + private $userManager; + /** @var IL10N */ + private $l10n; + public function setUp(): void { + $this->userManager = $this->createMock(IUserManager::class); + $this->urlGenerator = $this->createMock(IURLGenerator::class); + $this->l10n = $this->createMock(IL10N::class); + $this->service = new DeckJsonService( + $this->userManager, + $this->urlGenerator, + $this->l10n + ); + } + + public function testGetBoardWithNoName() { + $this->expectExceptionMessage('Invalid name of board'); + $importService = $this->createMock(BoardImportService::class); + $this->service->setImportService($importService); + $this->service->getBoard(); + } + + public function testGetBoardWithSuccess() { + $importService = Server::get(BoardImportService::class); + + $data = json_decode(file_get_contents(__DIR__ . '/../../../../data/deck.json')); + $importService->setData($data); + + $configInstance = json_decode(file_get_contents(__DIR__ . '/../../../../data/config-trelloJson.json')); + $importService->setConfigInstance($configInstance); + + $owner = $this->createMock(IUser::class); + $owner + ->method('getUID') + ->willReturn('owner'); + $importService->setConfig('owner', $owner); + + $this->service->setImportService($importService); + + $boards = $this->service->getBoards(); + $importService->setData($boards[0]); + $actual = $this->service->getBoard(); + $this->assertEquals('My test board', $actual->getTitle()); + $this->assertEquals('admin', $actual->getOwner()); + $this->assertEquals('e0ed31', $actual->getColor()); + } +} From 748f8bee3fe54b589e012d3595d45052dd9cd75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Tue, 18 Jul 2023 15:20:53 +0200 Subject: [PATCH 07/26] test: Add example test data for deck import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- tests/data/deck.json | 748 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 748 insertions(+) create mode 100644 tests/data/deck.json diff --git a/tests/data/deck.json b/tests/data/deck.json new file mode 100644 index 000000000..b6744ec94 --- /dev/null +++ b/tests/data/deck.json @@ -0,0 +1,748 @@ +{ + "version": "1.11.0-dev", + "boards": { + "188": { + "id": 188, + "title": "My test board", + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "color": "e0ed31", + "archived": false, + "labels": [ + { + "id": 239, + "title": "L2", + "color": "31CC7C", + "boardId": 188, + "cardId": null, + "lastModified": 1689667435, + "ETag": "63b77251cca5a56fe74a97e4baeab59c" + }, + { + "id": 240, + "title": "L4", + "color": "317CCC", + "boardId": 188, + "cardId": null, + "lastModified": 1689667442, + "ETag": "15dcabeb47583ce5398faaeb65f7a4b6" + }, + { + "id": 241, + "title": "L1", + "color": "FF7A66", + "boardId": 188, + "cardId": null, + "lastModified": 1689667432, + "ETag": "7d58be91f19ebc4f94b352db8c76c056" + }, + { + "id": 242, + "title": "L3", + "color": "F1DB50", + "boardId": 188, + "cardId": null, + "lastModified": 1689667440, + "ETag": "160253b9d33ae0a7a3af90e7d418ba60" + } + ], + "acl": [], + "permissions": { + "PERMISSION_READ": true, + "PERMISSION_EDIT": true, + "PERMISSION_MANAGE": true, + "PERMISSION_SHARE": true + }, + "users": [], + "shared": 0, + "stacks": { + "64": { + "id": 64, + "title": "A", + "boardId": 188, + "deletedAt": 0, + "lastModified": 1689667779, + "order": 999, + "ETag": "ddfd0c27e53d8db94ac5e9aaa021746e", + "cards": [ + { + "id": 114, + "title": "1", + "description": "", + "stackId": 64, + "type": "plain", + "lastModified": 1689667779, + "lastEditor": null, + "createdAt": 1689667569, + "labels": [ + { + "id": 239, + "title": "L2", + "color": "31CC7C", + "boardId": 188, + "cardId": 114, + "lastModified": 1689667435, + "ETag": "63b77251cca5a56fe74a97e4baeab59c" + } + ], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": "2050-07-24T22:00:00+00:00", + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "ddfd0c27e53d8db94ac5e9aaa021746e", + "overdue": 0, + "boardId": 188, + "board": { + "id": 188, + "title": "My test board" + } + }, + { + "id": 115, + "title": "2", + "description": "", + "stackId": 64, + "type": "plain", + "lastModified": 1689667752, + "lastEditor": null, + "createdAt": 1689667572, + "labels": [], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": "2023-07-17T02:00:00+00:00", + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "9a8ed495f7d83f8310ae6291d6dc4624", + "overdue": 3, + "boardId": 188, + "board": { + "id": 188, + "title": "My test board" + } + }, + { + "id": 116, + "title": "3", + "description": "", + "stackId": 64, + "type": "plain", + "lastModified": 1689667760, + "lastEditor": "admin", + "createdAt": 1689667576, + "labels": [], + "assignedUsers": [ + { + "id": 5, + "participant": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "cardId": 116, + "type": 0 + } + ], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "f908c4359e9ca0703f50da2bbe967594", + "overdue": 0, + "boardId": 188, + "board": { + "id": 188, + "title": "My test board" + } + } + ] + }, + "65": { + "id": 65, + "title": "B", + "boardId": 188, + "deletedAt": 0, + "lastModified": 1689667796, + "order": 999, + "ETag": "b97a2b19e1cafc8f95e3f4db71097214", + "cards": [ + { + "id": 117, + "title": "4", + "description": "", + "stackId": 65, + "type": "plain", + "lastModified": 1689667767, + "lastEditor": "admin", + "createdAt": 1689667578, + "labels": [ + { + "id": 239, + "title": "L2", + "color": "31CC7C", + "boardId": 188, + "cardId": 117, + "lastModified": 1689667435, + "ETag": "63b77251cca5a56fe74a97e4baeab59c" + }, + { + "id": 240, + "title": "L4", + "color": "317CCC", + "boardId": 188, + "cardId": 117, + "lastModified": 1689667442, + "ETag": "15dcabeb47583ce5398faaeb65f7a4b6" + }, + { + "id": 241, + "title": "L1", + "color": "FF7A66", + "boardId": 188, + "cardId": 117, + "lastModified": 1689667432, + "ETag": "7d58be91f19ebc4f94b352db8c76c056" + } + ], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "6b20cc46fa5d2e5f65251526b50cc130", + "overdue": 0, + "boardId": 188, + "board": { + "id": 188, + "title": "My test board" + } + }, + { + "id": 118, + "title": "5", + "description": "", + "stackId": 65, + "type": "plain", + "lastModified": 1689667773, + "lastEditor": "admin", + "createdAt": 1689667581, + "labels": [ + { + "id": 239, + "title": "L2", + "color": "31CC7C", + "boardId": 188, + "cardId": 118, + "lastModified": 1689667435, + "ETag": "63b77251cca5a56fe74a97e4baeab59c" + } + ], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "488145982535a91d9ab47db647ecf539", + "overdue": 0, + "boardId": 188, + "board": { + "id": 188, + "title": "My test board" + } + }, + { + "id": 119, + "title": "6", + "description": "# Test description\n\nHello world", + "stackId": 65, + "type": "plain", + "lastModified": 1689667796, + "lastEditor": null, + "createdAt": 1689667583, + "labels": [], + "assignedUsers": [ + { + "id": 6, + "participant": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "cardId": 119, + "type": 0 + } + ], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "b97a2b19e1cafc8f95e3f4db71097214", + "overdue": 0, + "boardId": 188, + "board": { + "id": 188, + "title": "My test board" + } + } + ] + }, + "66": { + "id": 66, + "title": "C", + "boardId": 188, + "deletedAt": 0, + "lastModified": 0, + "order": 999, + "ETag": "cfcd208495d565ef66e7dff9f98764da" + } + }, + "activeSessions": [], + "deletedAt": 0, + "lastModified": 1689667796, + "settings": [], + "ETag": "b97a2b19e1cafc8f95e3f4db71097214" + }, + "189": { + "id": 189, + "title": "Shared board", + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "color": "30b6d8", + "archived": false, + "labels": [ + { + "id": 243, + "title": "Finished", + "color": "31CC7C", + "boardId": 189, + "cardId": null, + "lastModified": 1689667413, + "ETag": "aa71367f6a9a2fc2d47fc46163a30208" + }, + { + "id": 244, + "title": "To review", + "color": "317CCC", + "boardId": 189, + "cardId": null, + "lastModified": 1689667413, + "ETag": "aa71367f6a9a2fc2d47fc46163a30208" + }, + { + "id": 245, + "title": "Action needed", + "color": "FF7A66", + "boardId": 189, + "cardId": null, + "lastModified": 1689667413, + "ETag": "aa71367f6a9a2fc2d47fc46163a30208" + }, + { + "id": 246, + "title": "Later", + "color": "F1DB50", + "boardId": 189, + "cardId": null, + "lastModified": 1689667413, + "ETag": "aa71367f6a9a2fc2d47fc46163a30208" + } + ], + "acl": [ + { + "id": 4, + "participant": { + "primaryKey": "alice", + "uid": "alice", + "displayname": "alice", + "type": 0 + }, + "type": 0, + "boardId": 189, + "permissionEdit": true, + "permissionShare": false, + "permissionManage": false, + "owner": false + }, + { + "id": 5, + "participant": { + "primaryKey": "jane", + "uid": "jane", + "displayname": "jane", + "type": 0 + }, + "type": 0, + "boardId": 189, + "permissionEdit": false, + "permissionShare": true, + "permissionManage": false, + "owner": false + }, + { + "id": 6, + "participant": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 1 + }, + "type": 1, + "boardId": 189, + "permissionEdit": false, + "permissionShare": false, + "permissionManage": true, + "owner": false + } + ], + "permissions": { + "PERMISSION_READ": true, + "PERMISSION_EDIT": true, + "PERMISSION_MANAGE": true, + "PERMISSION_SHARE": true + }, + "users": [], + "shared": 0, + "stacks": { + "61": { + "id": 61, + "title": "ToDo", + "boardId": 189, + "deletedAt": 0, + "lastModified": 1689667537, + "order": 999, + "ETag": "6c315c83f146485e6b2b6fdc24ffa617", + "cards": [ + { + "id": 107, + "title": "Write tests", + "description": "", + "stackId": 61, + "type": "plain", + "lastModified": 1689667521, + "lastEditor": null, + "createdAt": 1689667483, + "labels": [], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 0, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "f0450d41827f55580554c993304c8073", + "overdue": 0, + "boardId": 189, + "board": { + "id": 189, + "title": "Shared board" + } + }, + { + "id": 111, + "title": "Write blog post", + "description": "", + "stackId": 61, + "type": "plain", + "lastModified": 1689667521, + "lastEditor": null, + "createdAt": 1689667518, + "labels": [], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 1, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "f0450d41827f55580554c993304c8073", + "overdue": 0, + "boardId": 189, + "board": { + "id": 189, + "title": "Shared board" + } + }, + { + "id": 112, + "title": "Announce feature", + "description": "", + "stackId": 61, + "type": "plain", + "lastModified": 1689667527, + "lastEditor": null, + "createdAt": 1689667527, + "labels": [], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "1956848c45be91fefc967ee8831ea4cf", + "overdue": 0, + "boardId": 189, + "board": { + "id": 189, + "title": "Shared board" + } + }, + { + "id": 113, + "title": "\ud83c\udf89 Party", + "description": "", + "stackId": 61, + "type": "plain", + "lastModified": 1689667537, + "lastEditor": null, + "createdAt": 1689667537, + "labels": [], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "6c315c83f146485e6b2b6fdc24ffa617", + "overdue": 0, + "boardId": 189, + "board": { + "id": 189, + "title": "Shared board" + } + } + ] + }, + "62": { + "id": 62, + "title": "In progress", + "boardId": 189, + "deletedAt": 0, + "lastModified": 1689667502, + "order": 999, + "ETag": "1498939b8816e6041da80050dacc3ed3", + "cards": [ + { + "id": 108, + "title": "Write feature", + "description": "", + "stackId": 62, + "type": "plain", + "lastModified": 1689667488, + "lastEditor": null, + "createdAt": 1689667488, + "labels": [], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 999, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "d2a8b634cdd96ab5ef48910bbbd715b1", + "overdue": 0, + "boardId": 189, + "board": { + "id": 189, + "title": "Shared board" + } + } + ] + }, + "63": { + "id": 63, + "title": "Done", + "boardId": 189, + "deletedAt": 0, + "lastModified": 1689667518, + "order": 999, + "ETag": "09ba5a39921de760db53bcd56457eea5", + "cards": [ + { + "id": 109, + "title": "Plan feature", + "description": "", + "stackId": 63, + "type": "plain", + "lastModified": 1689667506, + "lastEditor": null, + "createdAt": 1689667493, + "labels": [], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 0, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "193163d8a8acedbfaba196b1f0d65bc8", + "overdue": 0, + "boardId": 189, + "board": { + "id": 189, + "title": "Shared board" + } + }, + { + "id": 110, + "title": "Design feature", + "description": "", + "stackId": 63, + "type": "plain", + "lastModified": 1689667506, + "lastEditor": null, + "createdAt": 1689667502, + "labels": [], + "assignedUsers": [], + "attachments": null, + "attachmentCount": null, + "owner": { + "primaryKey": "admin", + "uid": "admin", + "displayname": "admin", + "type": 0 + }, + "order": 1, + "archived": false, + "duedate": null, + "deletedAt": 0, + "commentsUnread": 0, + "commentsCount": 0, + "ETag": "193163d8a8acedbfaba196b1f0d65bc8", + "overdue": 0, + "boardId": 189, + "board": { + "id": 189, + "title": "Shared board" + } + } + ] + } + }, + "activeSessions": [], + "deletedAt": 0, + "lastModified": 1689667537, + "settings": [], + "ETag": "6c315c83f146485e6b2b6fdc24ffa617" + } + } +} From 890893f9e377d37d5b8439a289b55bada486562e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Wed, 19 Jul 2023 09:49:30 +0200 Subject: [PATCH 08/26] docs: Add dedicated documentation section for import/export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl docs: Update import docs Signed-off-by: Julius Härtl --- docs/export-import.md | 98 +++++++++++++++++++ .../Importer/BoardImportCommandService.php | 2 +- .../Importer/Systems/DeckJsonService.php | 4 +- .../Importer/Systems/DeckJsonServiceTest.php | 4 +- 4 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 docs/export-import.md diff --git a/docs/export-import.md b/docs/export-import.md new file mode 100644 index 000000000..ca7cd840a --- /dev/null +++ b/docs/export-import.md @@ -0,0 +1,98 @@ +## Export + +Deck currently supports exporting all boards a user owns in a single JSON file. The format is based on the database schema that deck uses. It can be used to re-import boards on the same or other instances. + +The export currently has some kown limitations in terms of specific data not included: +- Activity information +- File attachments to deck cards +- Comments +- +``` +occ deck:export > my-file.json +``` + +## Import boards + +Importing can be done using the API or the `occ` `deck:import` command. + +It is possible to import from the following sources: + +### Deck JSON + +A json file that has been obtained from the above described `occ deck:export [userid]` command can be imported. + +``` +occ deck:import my-file.json +``` + +In case you are importing from a different instance you may use an additional config file to provide custom user id mapping in case users have different identifiers. + +``` +{ + "owner": "admin", + "uidRelation": { + "johndoe": "test-user-1" + } +} +``` + +#### Trello JSON + +Limitations: +* Comments with more than 1000 characters are placed as attached files to the card. + +Steps: +* Create the data file + * Access Trello + * go to the board you want to export + * Follow the steps in [Trello documentation](https://help.trello.com/article/747-exporting-data-from-trello-1) and export as JSON +* Create the configuration file +* Execute the import informing the import file path, data file and source as `Trello JSON` + +Create the configuration file respecting the [JSON Schema](https://github.com/nextcloud/deck/blob/main/lib/Service/Importer/fixtures/config-trelloJson-schema.json) for import `Trello JSON` + +Example configuration file: +```json +{ + "owner": "admin", + "color": "0800fd", + "uidRelation": { + "johndoe": "johndoe" + } +} +``` + +**Limitations**: + +Importing from a JSON file imports up to 1000 actions. To find out how many actions the board to be imported has, identify how many actions the JSON has. + +#### Trello API + +Import using API is recommended for boards with more than 1000 actions. + +Trello makes it possible to attach links to a card. Deck does not have this feature. Attachments and attachment links are added in a markdown table at the end of the description for every imported card that has attachments in Trello. + +* Get the API Key and API Token [here](https://developer.atlassian.com/cloud/trello/guides/rest-api/api-introduction/#authentication-and-authorization) +* Get the ID of the board you want to import by making a request to: + https://api.trello.com/1/members/me/boards?key={yourKey}&token={yourToken}&fields=id,name + + This ID you will use in the configuration file in the `board` property +* Create the configuration file + +Create the configuration file respecting the [JSON Schema](https://github.com/nextcloud/deck/blob/main/lib/Service/Importer/fixtures/config-trelloApi-schema.json) for import `Trello JSON` + +Example configuration file: +```json +{ + "owner": "admin", + "color": "0800fd", + "api": { + "key": "0cc175b9c0f1b6a831c399e269772661", + "token": "92eb5ffee6ae2fec3ad71c777531578f4a8a08f09d37b73795649038408b5f33" + }, + "board": "8277e0910d750195b4487976", + "uidRelation": { + "johndoe": "johndoe" + } +} +``` diff --git a/lib/Service/Importer/BoardImportCommandService.php b/lib/Service/Importer/BoardImportCommandService.php index 0aea5ca97..e599f7ff6 100644 --- a/lib/Service/Importer/BoardImportCommandService.php +++ b/lib/Service/Importer/BoardImportCommandService.php @@ -184,7 +184,7 @@ public function import(): void { foreach ($boards as $board) { $this->reset(); $this->setData($board); - $this->getOutput()->writeln('Importing board...'); + $this->getOutput()->writeln('Importing board "' . $this->getBoard()->getTitle() . '".'); $this->importBoard(); $this->getOutput()->writeln('Assign users to board...'); $this->importAcl(); diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index 03d3c2e6e..703068482 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -1,8 +1,8 @@ + * @copyright Copyright (c) 2023 Julius Härtl * - * @author Vitor Mattos + * @author Julius Härtl * * @license GNU AGPL version 3 or any later version * diff --git a/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php b/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php index 1a2c41aa1..fb476ecc9 100644 --- a/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php +++ b/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php @@ -1,8 +1,8 @@ + * @copyright Copyright (c) 2023 Julius Härtl * - * @author Vitor Mattos + * @author Julius Härtl * * @license GNU AGPL version 3 or any later version * From f5648284f2334b9753cc36782730c0b4ec7de8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Wed, 19 Jul 2023 09:52:44 +0200 Subject: [PATCH 09/26] feat: Let occ deck:import default to deck json importer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Command/BoardImport.php | 12 ++- lib/Db/RelationalEntity.php | 2 +- .../Importer/BoardImportCommandService.php | 17 ++++ lib/Service/Importer/BoardImportService.php | 12 ++- .../Importer/Systems/DeckJsonService.php | 19 +++- tests/integration/import/ImportExportTest.php | 93 ++++++++++++++++++- .../Importer/BoardImportServiceTest.php | 8 +- 7 files changed, 146 insertions(+), 17 deletions(-) diff --git a/lib/Command/BoardImport.php b/lib/Command/BoardImport.php index 4c70c61ae..66ee33b3e 100644 --- a/lib/Command/BoardImport.php +++ b/lib/Command/BoardImport.php @@ -25,6 +25,7 @@ use OCA\Deck\Service\Importer\BoardImportCommandService; use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; @@ -41,7 +42,9 @@ public function __construct( */ protected function configure() { $allowedSystems = $this->boardImportCommandService->getAllowedImportSystems(); - $names = array_column($allowedSystems, 'name'); + $names = array_map(function ($name) { + return '"' . $name . '"'; + }, array_column($allowedSystems, 'internalName')); $this ->setName('deck:import') ->setDescription('Import data') @@ -50,7 +53,7 @@ protected function configure() { null, InputOption::VALUE_REQUIRED, 'Source system for import. Available options: ' . implode(', ', $names) . '.', - null + 'DeckJson', ) ->addOption( 'config', @@ -66,6 +69,11 @@ protected function configure() { 'Data file to import.', 'data.json' ) + ->addArgument( + 'file', + InputArgument::OPTIONAL, + 'File to import', + ) ; } diff --git a/lib/Db/RelationalEntity.php b/lib/Db/RelationalEntity.php index 683f1611f..76652ccad 100644 --- a/lib/Db/RelationalEntity.php +++ b/lib/Db/RelationalEntity.php @@ -138,7 +138,7 @@ public function __call(string $methodName, array $args) { $attr = lcfirst(substr($methodName, 3)); if (array_key_exists($attr, $this->_resolvedProperties) && strpos($methodName, 'set') === 0) { - if (!is_scalar($args[0])) { + if ($args[0] !== null && !is_scalar($args[0])) { $args[0] = $args[0]['primaryKey']; } parent::setter($attr, $args); diff --git a/lib/Service/Importer/BoardImportCommandService.php b/lib/Service/Importer/BoardImportCommandService.php index e599f7ff6..23257e499 100644 --- a/lib/Service/Importer/BoardImportCommandService.php +++ b/lib/Service/Importer/BoardImportCommandService.php @@ -25,6 +25,7 @@ use OCA\Deck\Exceptions\ConflictException; use OCA\Deck\NotFoundException; +use OCA\Deck\Service\Importer\Systems\DeckJsonService; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -76,6 +77,10 @@ public function getOutput(): OutputInterface { } protected function validateConfig(): void { + // FIXME: Make config optional for deck plain importer (but use a call on the importer insterad) + if ($this->getImportSystem() instanceof DeckJsonService) { + return; + } try { $config = $this->getInput()->getOption('config'); if (is_string($config)) { @@ -145,6 +150,18 @@ protected function validateData(): void { if (!$this->getImportSystem()->needValidateData()) { return; } + $data = $this->getInput()->getArgument('file'); + if (is_string($data)) { + if (!file_exists($data)) { + throw new \OCP\Files\NotFoundException('Could not find file ' . $data); + } + $data = json_decode(file_get_contents($data)); + if ($data instanceof \stdClass) { + $this->setData($data); + return; + } + } + $data = $this->getInput()->getOption('data'); if (is_string($data)) { $data = json_decode(file_get_contents($data)); diff --git a/lib/Service/Importer/BoardImportService.php b/lib/Service/Importer/BoardImportService.php index 364fd1bce..dcd0bbc0c 100644 --- a/lib/Service/Importer/BoardImportService.php +++ b/lib/Service/Importer/BoardImportService.php @@ -84,6 +84,8 @@ public function __construct( ) { $this->board = new Board(); $this->disableCommentsEvents(); + + $this->config = new \stdClass(); } private function disableCommentsEvents(): void { @@ -151,6 +153,11 @@ public function addAllowedImportSystem($system): self { public function getAllowedImportSystems(): array { if (!$this->allowedSystems) { + $this->addAllowedImportSystem([ + 'name' => DeckJsonService::$name, + 'class' => DeckJsonService::class, + 'internalName' => 'DeckJson' + ]); $this->addAllowedImportSystem([ 'name' => TrelloApiService::$name, 'class' => TrelloApiService::class, @@ -161,11 +168,6 @@ public function getAllowedImportSystems(): array { 'class' => TrelloJsonService::class, 'internalName' => 'TrelloJson' ]); - $this->addAllowedImportSystem([ - 'name' => DeckJsonService::$name, - 'class' => DeckJsonService::class, - 'internalName' => 'DeckJson' - ]); } $this->eventDispatcher->dispatchTyped(new BoardImportGetAllowedEvent($this)); return $this->allowedSystems; diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index 703068482..57caff6c7 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -31,8 +31,6 @@ use OCA\Deck\Db\Label; use OCA\Deck\Db\Stack; use OCA\Deck\Service\Importer\ABoardImportService; -use OCP\IL10N; -use OCP\IURLGenerator; use OCP\IUser; use OCP\IUserManager; @@ -44,8 +42,6 @@ class DeckJsonService extends ABoardImportService { public function __construct( private IUserManager $userManager, - private IURLGenerator $urlGenerator, - private IL10N $l10n ) { } @@ -86,6 +82,20 @@ public function validateUsers(): void { } } + public function mapMember($uid): ?string { + + $uidCandidate = $this->members[$uid]?->getUID() ?? null; + if ($uidCandidate) { + return $uidCandidate; + } + + if ($this->userManager->userExists($uid)) { + return $uid; + } + + return null; + } + public function getCardAssignments(): array { $assignments = []; foreach ($this->tmpCards as $sourceCard) { @@ -176,6 +186,7 @@ public function getCards(): array { $card = new Card(); $card->setTitle($cardSource->title); $card->setLastModified($cardSource->lastModified); + $card->setCreatedAt($cardSource->createdAt); $card->setArchived($cardSource->archived); $card->setDescription($cardSource->description); $card->setStackId($this->stacks[$cardSource->stackId]->getId()); diff --git a/tests/integration/import/ImportExportTest.php b/tests/integration/import/ImportExportTest.php index f6e339d6a..4ee73c5ad 100644 --- a/tests/integration/import/ImportExportTest.php +++ b/tests/integration/import/ImportExportTest.php @@ -26,6 +26,7 @@ use OCA\Deck\Command\BoardImport; use OCA\Deck\Service\Importer\BoardImportService; use OCA\Deck\Service\Importer\Systems\DeckJsonService; +use OCP\AppFramework\Db\Entity; use OCP\IDBConnection; use OCP\IGroupManager; use OCP\IUserManager; @@ -117,10 +118,98 @@ public function testImport() { public function assertDatabase() { $boardMapper = \OCP\Server::get(BoardMapper::class); + $stackMapper = \OCP\Server::get(StackMapper::class); + $cardMapper = \OCP\Server::get(CardMapper::class); + $boards = $boardMapper->findAllByOwner('admin'); - self::assertEquals('My test board', $boards[0]->getTitle()); - self::assertEquals('Shared board', $boards[1]->getTitle()); self::assertEquals(2, count($boards)); + + $board = $boards[0]; + self::assertEntity(Board::fromRow([ + 'title' => 'My test board', + 'color' => 'e0ed31', + 'owner' => 'admin', + ]), $board); + + $stacks = $stackMapper->findAll($board->getId()); + self::assertCount(3, $stacks); + self::assertEntity(Stack::fromRow([ + 'title' => 'A', + 'order' => 999, + 'boardId' => $boards[0]->getId(), + ]), $stacks[0]); + self::assertEntity(Stack::fromRow([ + 'title' => 'B', + 'order' => 999, + 'boardId' => $boards[0]->getId(), + ]), $stacks[1]); + self::assertEntity(Stack::fromRow([ + 'title' => 'C', + 'order' => 999, + 'boardId' => $boards[0]->getId(), + ]), $stacks[2]); + + $cards = $cardMapper->findAll($stacks[0]->getId()); + self::assertEntity(Card::fromRow([ + 'title' => '1', + 'description' => '', + 'type' => 'plain', + 'lastModified' => 1689667779, + 'createdAt' => 1689667569, + 'owner' => 'admin', + 'duedate' => new \DateTime('2050-07-24T22:00:00.000000+0000'), + 'order' => 999, + 'stackId' => $stacks[0]->getId(), + ]), $cards[0]); + self::assertEntity(Card::fromRow([ + 'title' => '2', + 'duedate' => new \DateTime('2050-07-24T22:00:00.000000+0000'), + ]), $cards[1], true); + self::assertEntity(Card::fromParams([ + 'title' => '3', + 'duedate' => null, + ]), $cards[2], true); + + $cards = $cardMapper->findAll($stacks[1]->getId()); + self::assertEntity(Card::fromParams([ + 'title' => '6', + 'duedate' => null, + 'description' => "# Test description\n\nHello world", + ]), $cards[2], true); + + // Shared board + $sharedBoard = $boards[1]; + self::assertEntity(Board::fromRow([ + 'title' => 'Shared board', + 'color' => '30b6d8', + 'owner' => 'admin', + ]), $sharedBoard); + + $stacks = $stackMapper->findAll($sharedBoard->getId()); + self::assertCount(3, $stacks); + } + + public static function assertEntity(Entity $expected, Entity $actual, bool $checkProperties = false) { + if ($checkProperties === true) { + $e = clone $expected; + $a = clone $actual; + foreach ($e->getUpdatedFields() as $property => $updated) { + $expectedValue = call_user_func([$e, 'get' . ucfirst($property)]); + $actualValue = call_user_func([$a, 'get' . ucfirst($property)]); + self::assertEquals( + $expectedValue, + $actualValue + ); + } + } else { + $e = clone $expected; + $e->setId(null); + $a = clone $actual; + $a->setId(null); + $e->resetUpdatedFields(); + $a->resetUpdatedFields(); + self::assertEquals($e, $a); + } } public function tearDown(): void { diff --git a/tests/unit/Service/Importer/BoardImportServiceTest.php b/tests/unit/Service/Importer/BoardImportServiceTest.php index 9e4229529..c652599b0 100644 --- a/tests/unit/Service/Importer/BoardImportServiceTest.php +++ b/tests/unit/Service/Importer/BoardImportServiceTest.php @@ -118,6 +118,9 @@ public function setUp(): void { $this->trelloJsonService ->method('getJsonSchemaPath') ->willReturn($configFile); + $this->trelloJsonService + ->method('getBoards') + ->willReturn([$data]); $this->boardImportService->setImportSystem($this->trelloJsonService); $owner = $this->createMock(IUser::class); @@ -192,8 +195,7 @@ public function testImportSuccess() { ->expects($this->once()) ->method('insert'); - $actual = $this->boardImportService->import(); - - $this->assertNull($actual); + $this->boardImportService->import(); + self::assertTrue(true); } } From f160ec63c0c63bdb50ac6303689ca623ef0217e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Wed, 19 Jul 2023 09:55:43 +0200 Subject: [PATCH 10/26] draft: todos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Service/Importer/BoardImportService.php | 1 - lib/Service/Importer/Systems/DeckJsonService.php | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Service/Importer/BoardImportService.php b/lib/Service/Importer/BoardImportService.php index dcd0bbc0c..ba08c7a93 100644 --- a/lib/Service/Importer/BoardImportService.php +++ b/lib/Service/Importer/BoardImportService.php @@ -117,7 +117,6 @@ public function import(): void { $this->importComments(); $this->importCardAssignments(); } catch (\Throwable $th) { - throw $th; throw new BadRequestException($th->getMessage()); } } diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index 57caff6c7..37c99ec11 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -216,6 +216,8 @@ public function getAclList(): array { $acl->setPermissionEdit(false); $acl->setPermissionShare(false); $acl->setPermissionManage(false); + // FIXME: Figure out a way to collect and aggregate warnings about users + // FIXME: Maybe have a dry run? $return[] = $acl; } return $return; From f98873093dff8f41dc090edb27ebfb50db1eaff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Thu, 20 Jul 2023 10:54:33 +0200 Subject: [PATCH 11/26] fix: request full details for board export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Command/UserExport.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Command/UserExport.php b/lib/Command/UserExport.php index 4d869e275..b435e2ee5 100644 --- a/lib/Command/UserExport.php +++ b/lib/Command/UserExport.php @@ -69,7 +69,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $legacyFormat = $input->getOption('legacy-format'); $this->boardService->setUserId($userId); - $boards = $this->boardService->findAll(); + $boards = $this->boardService->findAll(fullDetails: true); $data = []; foreach ($boards as $board) { From 073093c9589561f93de25333d0dff1e1b28275d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Thu, 20 Jul 2023 10:55:38 +0200 Subject: [PATCH 12/26] test: Add reimport test case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- .../Importer/Systems/DeckJsonService.php | 31 ++--- tests/integration/import/ImportExportTest.php | 125 +++++++++++++++++- 2 files changed, 139 insertions(+), 17 deletions(-) diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index 37c99ec11..b9d202b3a 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -132,9 +132,13 @@ public function getBoard(): Board { if (empty($this->getImportService()->getData()->title)) { throw new BadRequestException('Invalid name of board'); } - $board->setTitle($this->getImportService()->getData()->title); - $board->setOwner($this->getImportService()->getData()->owner->uid); - $board->setColor($this->getImportService()->getData()->color); + $importBoard = $this->getImportService()->getData(); + $board->setTitle($importBoard->title); + $board->setOwner($importBoard->owner->uid); + $board->setColor($importBoard->color); + $board->setArchived($importBoard->archived); + $board->setDeletedAt($importBoard->deletedAt); + $board->setLastModified($importBoard->lastModified); return $board; } @@ -147,6 +151,7 @@ public function getLabels(): array { $newLabel->setTitle($label->title); $newLabel->setColor($label->color); $newLabel->setBoardId($this->getImportService()->getBoard()->getId()); + $newLabel->setLastModified($label->lastModified); $this->labels[$label->id] = $newLabel; } return $this->labels; @@ -203,21 +208,17 @@ public function getCards(): array { * @return Acl[] */ public function getAclList(): array { - // FIXME: To implement + $board = $this->getImportService()->getData(); $return = []; - foreach ($this->members as $member) { - if ($member->getUID() === $this->getImportService()->getConfig('owner')->getUID()) { - continue; - } + foreach ($board->acl as $aclData) { + // FIXME: Figure out mapping $acl = new Acl(); $acl->setBoardId($this->getImportService()->getBoard()->getId()); - $acl->setType(Acl::PERMISSION_TYPE_USER); - $acl->setParticipant($member->getUID()); - $acl->setPermissionEdit(false); - $acl->setPermissionShare(false); - $acl->setPermissionManage(false); - // FIXME: Figure out a way to collect and aggregate warnings about users - // FIXME: Maybe have a dry run? + $acl->setType($aclData->type); + $acl->setParticipant($aclData->participant?->primaryKey ?? $aclData->participant); + $acl->setPermissionEdit($aclData->permissionEdit); + $acl->setPermissionShare($aclData->permissionShare); + $acl->setPermissionManage($aclData->permissionManage); $return[] = $acl; } return $return; diff --git a/tests/integration/import/ImportExportTest.php b/tests/integration/import/ImportExportTest.php index 4ee73c5ad..ce79fe1fe 100644 --- a/tests/integration/import/ImportExportTest.php +++ b/tests/integration/import/ImportExportTest.php @@ -24,15 +24,21 @@ namespace OCA\Deck\Db; use OCA\Deck\Command\BoardImport; +use OCA\Deck\Command\UserExport; +use OCA\Deck\Service\BoardService; use OCA\Deck\Service\Importer\BoardImportService; use OCA\Deck\Service\Importer\Systems\DeckJsonService; +use OCA\Deck\Service\PermissionService; +use OCA\Deck\Service\StackService; use OCP\AppFramework\Db\Entity; use OCP\IDBConnection; use OCP\IGroupManager; use OCP\IUserManager; use OCP\Server; +use PHPUnit\Framework\ExpectationFailedException; use Symfony\Component\Console\Application; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\BufferedOutput; use Symfony\Component\Console\Output\OutputInterface; /** @@ -102,6 +108,73 @@ public function testImportOcc() { $this->assertDatabase(); } + /** + * This test runs an import, export and another import to assert that multiple attempts result in the same data structure + */ + public function testReimportOcc() { + // initial import from test fixture json + $input = $this->createMock(InputInterface::class); + $input->expects($this->any()) + ->method('getOption') + ->willReturnCallback(function ($arg) { + return match ($arg) { + 'system' => 'DeckJson', + 'data' => __DIR__ . '/../../data/deck.json', + 'config' => __DIR__ . '/../../data/config-trelloJson.json', + }; + }); + $output = $this->createMock(OutputInterface::class); + $importer = \OCP\Server::get(BoardImport::class); + $application = new Application(); + $importer->setApplication($application); + $importer->run($input, $output); + + $this->assertDatabase(); + + self::overwriteService(BoardService::class, self::getFreshService(BoardService::class)); + // export to a temporary file + $input = $this->createMock(InputInterface::class); + $input->expects($this->any()) + ->method('getArgument') + ->with('user-id') + ->willReturn('admin'); + $output = new BufferedOutput(); + $exporter = \OCP\Server::get(UserExport::class); + $exporter->setApplication($application); + $exporter->run($input, $output); + $jsonOutput = $output->fetch(); + json_decode($jsonOutput); + self::assertTrue(json_last_error() === JSON_ERROR_NONE); + $tmpExportFile = tempnam('/tmp', 'export'); + file_put_contents($tmpExportFile, $jsonOutput); + + // self::assertEquals(file_get_contents(__DIR__ . '/../../data/deck.json'), $jsonOutput); + + // cleanup test database + $this->connection->rollBack(); + $this->connection->beginTransaction(); + + self::overwriteService(BoardService::class, self::getFreshService(BoardService::class)); + // Re-import from temporary file + $input = $this->createMock(InputInterface::class); + $input->expects($this->any()) + ->method('getOption') + ->willReturnCallback(function ($arg) use ($tmpExportFile) { + return match ($arg) { + 'system' => 'DeckJson', + 'data' => $tmpExportFile, + 'config' => __DIR__ . '/../../data/config-trelloJson.json', + }; + }); + $output = $this->createMock(OutputInterface::class); + $importer = \OCP\Server::get(BoardImport::class); + $application = new Application(); + $importer->setApplication($application); + $importer->run($input, $output); + + $this->assertDatabase(); + } + public function testImport() { $importer = \OCP\Server::get(BoardImportService::class); $deckJsonService = \OCP\Server::get(DeckJsonService::class); @@ -116,12 +189,24 @@ public function testImport() { $this->assertDatabase(); } + /** + * @template T + * @param class-string|string $className + * @return T + */ + private function getFreshService(string $className): mixed { + return \OC::$server->getRegisteredAppContainer('deck')->resolve($className); + } + public function assertDatabase() { + $permissionService = \OCP\Server::get(PermissionService::class); + $permissionService->setUserId('admin'); $boardMapper = \OCP\Server::get(BoardMapper::class); $stackMapper = \OCP\Server::get(StackMapper::class); $cardMapper = \OCP\Server::get(CardMapper::class); $boards = $boardMapper->findAllByOwner('admin'); + $boardNames = array_map(fn ($board) => $board->getTitle(), $boards); self::assertEquals(2, count($boards)); $board = $boards[0]; @@ -129,7 +214,15 @@ public function assertDatabase() { 'title' => 'My test board', 'color' => 'e0ed31', 'owner' => 'admin', + 'lastModified' => 1689667796, ]), $board); + $boardService = $this->getFreshService(BoardService::class); + $fullBoard = $boardService->find($board->getId(), true); + self::assertEntityInArray(Label::fromParams([ + 'title' => 'L2', + 'color' => '31CC7C', + ]), $fullBoard->getLabels(), true); + $stacks = $stackMapper->findAll($board->getId()); self::assertCount(3, $stacks); @@ -183,13 +276,41 @@ public function assertDatabase() { 'title' => 'Shared board', 'color' => '30b6d8', 'owner' => 'admin', - ]), $sharedBoard); + ]), $sharedBoard, true); + + $stackService = \OCP\Server::get(StackService::class); + $stacks = $stackService->findAll($board->getId()); + self::assertEntityInArray(Label::fromParams([ + 'title' => 'L2', + 'color' => '31CC7C', + ]), $stacks[0]->getCards()[0]->getLabels(), true); + self::assertEntity(Label::fromParams([ + 'title' => 'L2', + 'color' => '31CC7C', + ]), $stacks[0]->getCards()[0]->getLabels()[0], true); $stacks = $stackMapper->findAll($sharedBoard->getId()); self::assertCount(3, $stacks); } - public static function assertEntity(Entity $expected, Entity $actual, bool $checkProperties = false) { + public static function assertEntityInArray(Entity $expected, array $array, bool $checkProperties): void { + $exists = null; + foreach ($array as $entity) { + try { + self::assertEntity($expected, $entity, $checkProperties); + $exists = $entity; + } catch (ExpectationFailedException $e) { + } + } + if ($exists) { + self::assertEntity($expected, $exists, $checkProperties); + } else { + // THis is hard to debug if it fails as the actual diff is not returned but hidden in the above exception + self::assertEquals($expected, $exists); + } + } + + public static function assertEntity(Entity $expected, Entity $actual, bool $checkProperties = false): void { if ($checkProperties === true) { $e = clone $expected; $a = clone $actual; From fd9883207eba3c7fc72c3cba0b7fc1e300dc202b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Thu, 27 Jul 2023 20:36:25 +0200 Subject: [PATCH 13/26] fix: Avoid duplicate data on board export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Command/UserExport.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Command/UserExport.php b/lib/Command/UserExport.php index b435e2ee5..7355cb880 100644 --- a/lib/Command/UserExport.php +++ b/lib/Command/UserExport.php @@ -69,7 +69,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $legacyFormat = $input->getOption('legacy-format'); $this->boardService->setUserId($userId); - $boards = $this->boardService->findAll(fullDetails: true); + $boards = $this->boardService->findAll(fullDetails: false); $data = []; foreach ($boards as $board) { From 9ca442f959f3c78ec9957ce7b3c2d50df6d13609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Thu, 27 Jul 2023 20:37:02 +0200 Subject: [PATCH 14/26] fix: Only set last modified if not already set manually MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Db/LabelMapper.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/Db/LabelMapper.php b/lib/Db/LabelMapper.php index aeccef539..66fcfa0dd 100644 --- a/lib/Db/LabelMapper.php +++ b/lib/Db/LabelMapper.php @@ -115,7 +115,9 @@ public function findAssignedLabelsForBoard($boardId, $limit = null, $offset = nu } public function insert(Entity $entity): Entity { - $entity->setLastModified(time()); + if (!in_array('lastModified', $entity->getUpdatedFields())) { + $entity->setLastModified(time()); + } return parent::insert($entity); } From b78f8a664f021dd2b82fc69500acc496c73fec1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Thu, 27 Jul 2023 20:37:49 +0200 Subject: [PATCH 15/26] fix: Do not fail on missing owner details MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Service/Importer/Systems/DeckJsonService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index b9d202b3a..cf9b71d04 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -134,7 +134,7 @@ public function getBoard(): Board { } $importBoard = $this->getImportService()->getData(); $board->setTitle($importBoard->title); - $board->setOwner($importBoard->owner->uid); + $board->setOwner($importBoard->owner?->uid ?? $importBoard->owner); $board->setColor($importBoard->color); $board->setArchived($importBoard->archived); $board->setDeletedAt($importBoard->deletedAt); From 60df897bd22360d361f2920a782eac351799df24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Thu, 27 Jul 2023 20:38:13 +0200 Subject: [PATCH 16/26] tests: assert json diff between import/export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Db/BoardMapper.php | 4 +- .../Importer/Systems/DeckJsonService.php | 44 ++-- .../fixtures/config-deckJson-schema.json | 9 +- tests/data/config-deckJson.json | 7 + tests/integration/import/ImportExportTest.php | 214 ++++++++++++------ .../Importer/Systems/DeckJsonServiceTest.php | 4 +- 6 files changed, 184 insertions(+), 98 deletions(-) create mode 100644 tests/data/config-deckJson.json diff --git a/lib/Db/BoardMapper.php b/lib/Db/BoardMapper.php index 8f44c003b..e4a04a9ac 100644 --- a/lib/Db/BoardMapper.php +++ b/lib/Db/BoardMapper.php @@ -532,12 +532,12 @@ public function flushCache(?int $boardId = null, ?string $userId = null) { if ($boardId) { unset($this->boardCache[$boardId]); } else { - $this->boardCache = null; + $this->boardCache = new CappedMemoryCache(); } if ($userId) { unset($this->userBoardCache[$userId]); } else { - $this->userBoardCache = null; + $this->userBoardCache = new CappedMemoryCache(); } } } diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index cf9b71d04..d2c622154 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -59,16 +59,11 @@ public function getJsonSchemaPath(): string { } public function validateUsers(): void { - if (empty($this->getImportService()->getConfig('uidRelation')) || !isset($this->getImportService()->getData()->members)) { + $relation = $this->getImportService()->getConfig('uidRelation'); + if (empty($relation)) { return; } - foreach ($this->getImportService()->getConfig('uidRelation') as $exportUid => $nextcloudUid) { - $user = array_filter($this->getImportService()->getData()->members, function (\stdClass $u) use ($exportUid) { - return $u->username === $exportUid; - }); - if (!$user) { - throw new \LogicException('Trello user ' . $exportUid . ' not found in property "members" of json data'); - } + foreach ($relation as $exportUid => $nextcloudUid) { if (!is_string($nextcloudUid) && !is_numeric($nextcloudUid)) { throw new \LogicException('User on setting uidRelation is invalid'); } @@ -77,14 +72,12 @@ public function validateUsers(): void { if (!$this->getImportService()->getConfig('uidRelation')->$exportUid) { throw new \LogicException('User on setting uidRelation not found: ' . $nextcloudUid); } - $user = current($user); - $this->members[$user->id] = $this->getImportService()->getConfig('uidRelation')->$exportUid; + $this->members[$exportUid] = $this->getImportService()->getConfig('uidRelation')->$exportUid; } } public function mapMember($uid): ?string { - - $uidCandidate = $this->members[$uid]?->getUID() ?? null; + $uidCandidate = isset($this->members[$uid]) ? $this->members[$uid]?->getUID() ?? null : null; if ($uidCandidate) { return $uidCandidate; } @@ -96,6 +89,15 @@ public function mapMember($uid): ?string { return null; } + public function mapOwner(string $uid): string { + $configOwner = $this->getImportService()->getConfig('owner'); + if ($configOwner) { + return $configOwner->getUID(); + } + + return $uid; + } + public function getCardAssignments(): array { $assignments = []; foreach ($this->tmpCards as $sourceCard) { @@ -134,7 +136,7 @@ public function getBoard(): Board { } $importBoard = $this->getImportService()->getData(); $board->setTitle($importBoard->title); - $board->setOwner($importBoard->owner?->uid ?? $importBoard->owner); + $board->setOwner($this->mapOwner($importBoard->owner?->uid ?? $importBoard->owner)); $board->setColor($importBoard->color); $board->setArchived($importBoard->archived); $board->setDeletedAt($importBoard->deletedAt); @@ -168,6 +170,7 @@ public function getStacks(): array { $stack->setTitle($source->title); $stack->setBoardId($this->getImportService()->getBoard()->getId()); $stack->setOrder($source->order); + $stack->setLastModified($source->lastModified); $return[$source->id] = $stack; } @@ -191,13 +194,15 @@ public function getCards(): array { $card = new Card(); $card->setTitle($cardSource->title); $card->setLastModified($cardSource->lastModified); + $card->setLastEditor($cardSource->lastEditor); $card->setCreatedAt($cardSource->createdAt); $card->setArchived($cardSource->archived); $card->setDescription($cardSource->description); $card->setStackId($this->stacks[$cardSource->stackId]->getId()); $card->setType('plain'); $card->setOrder($cardSource->order); - $card->setOwner($this->getBoard()->getOwner()); + $boardOwner = $this->getBoard()->getOwner(); + $card->setOwner($this->mapOwner(is_string($boardOwner) ? $boardOwner : $boardOwner->getUID())); $card->setDuedate($cardSource->duedate); $cards[$cardSource->id] = $card; } @@ -215,11 +220,18 @@ public function getAclList(): array { $acl = new Acl(); $acl->setBoardId($this->getImportService()->getBoard()->getId()); $acl->setType($aclData->type); - $acl->setParticipant($aclData->participant?->primaryKey ?? $aclData->participant); + $participant = $aclData->participant?->primaryKey ?? $aclData->participant; + if ($acl->getType() === Acl::PERMISSION_TYPE_USER) { + $participant = $this->mapMember($participant); + } + $acl->setParticipant($participant); $acl->setPermissionEdit($aclData->permissionEdit); $acl->setPermissionShare($aclData->permissionShare); $acl->setPermissionManage($aclData->permissionManage); - $return[] = $acl; + if ($participant) { + $return[] = $acl; + } + // TODO: Once we have error collection we should catch non-existing users } return $return; } diff --git a/lib/Service/Importer/fixtures/config-deckJson-schema.json b/lib/Service/Importer/fixtures/config-deckJson-schema.json index 7635727c1..5a8024fad 100644 --- a/lib/Service/Importer/fixtures/config-deckJson-schema.json +++ b/lib/Service/Importer/fixtures/config-deckJson-schema.json @@ -12,13 +12,6 @@ "type": "string", "required": true, "comment": "Nextcloud owner username" - }, - "color": { - "type": "string", - "required": true, - "pattern": "^[0-9a-fA-F]{6}$", - "comment": "Default color for the board. If you don't inform, the default color will be used.", - "default": "0800fd" } } -} \ No newline at end of file +} diff --git a/tests/data/config-deckJson.json b/tests/data/config-deckJson.json new file mode 100644 index 000000000..ebb67200e --- /dev/null +++ b/tests/data/config-deckJson.json @@ -0,0 +1,7 @@ +{ + "owner": "admin", + "color": "0800fd", + "uidRelation": { + "johndoe": "test-user-1" + } +} diff --git a/tests/integration/import/ImportExportTest.php b/tests/integration/import/ImportExportTest.php index ce79fe1fe..64c14207a 100644 --- a/tests/integration/import/ImportExportTest.php +++ b/tests/integration/import/ImportExportTest.php @@ -26,10 +26,12 @@ use OCA\Deck\Command\BoardImport; use OCA\Deck\Command\UserExport; use OCA\Deck\Service\BoardService; +use OCA\Deck\Service\CardService; use OCA\Deck\Service\Importer\BoardImportService; use OCA\Deck\Service\Importer\Systems\DeckJsonService; use OCA\Deck\Service\PermissionService; use OCA\Deck\Service\StackService; +use OCP\App\IAppManager; use OCP\AppFramework\Db\Entity; use OCP\IDBConnection; use OCP\IGroupManager; @@ -59,6 +61,9 @@ public static function setUpBeforeClass(): void { $backend = new \Test\Util\User\Dummy(); \OC_User::useBackend($backend); Server::get(IUserManager::class)->registerBackend($backend); + $backend->createUser('alice', 'alice'); + $backend->createUser('jane', 'jane'); + $backend->createUser('johndoe', 'johndoe'); $backend->createUser(self::TEST_USER1, self::TEST_USER1); $backend->createUser(self::TEST_USER2, self::TEST_USER2); $backend->createUser(self::TEST_USER3, self::TEST_USER3); @@ -78,68 +83,118 @@ public static function setUpBeforeClass(): void { $groupBackend->addToGroup(self::TEST_USER4, 'group3'); $groupBackend->addToGroup(self::TEST_USER2, self::TEST_GROUP1); Server::get(IGroupManager::class)->addBackend($groupBackend); + + Server::get(PermissionService::class)->setUserId('admin'); } public function setUp(): void { parent::setUp(); $this->connection = \OCP\Server::get(IDBConnection::class); - $this->connection->beginTransaction(); - + $this->cleanDb(); + $this->cleanDb(self::TEST_USER1); } public function testImportOcc() { - $input = $this->createMock(InputInterface::class); - $input->expects($this->any()) - ->method('getOption') - ->willReturnCallback(function ($arg) { - return match ($arg) { - 'system' => 'DeckJson', - 'data' => __DIR__ . '/../../data/deck.json', - 'config' => __DIR__ . '/../../data/config-trelloJson.json', - }; - }); - $output = $this->createMock(OutputInterface::class); - $importer = \OCP\Server::get(BoardImport::class); - $application = new Application(); - $importer->setApplication($application); - $importer->run($input, $output); - + $this->importFromFile(__DIR__ . '/../../data/deck.json'); $this->assertDatabase(); } /** - * This test runs an import, export and another import to assert that multiple attempts result in the same data structure + * This test runs an import, export and another import and + * assert that multiple attempts result in the same data structure + * + * In addition, it asserts that multiple import/export runs result in the same JSON */ public function testReimportOcc() { - // initial import from test fixture json + $this->importFromFile(__DIR__ . '/../../data/deck.json'); + $this->assertDatabase(); + + $tmpExportFile = $this->exportToTemp(); + + // Useful for double checking differences as there is no easy way to compare equal with skipping certain id keys, etag + // self::assertEquals(file_get_contents(__DIR__ . '/../../data/deck.json'), $jsonOutput); + self::assertEquals( + self::writeArrayStructure(array: json_decode(file_get_contents(__DIR__ . '/../../data/deck.json'), true)), + self::writeArrayStructure(array: json_decode(file_get_contents($tmpExportFile), true)) + ); + + // cleanup test database + $this->cleanDb(); + + // Re-import from temporary file + $this->importFromFile($tmpExportFile); + $this->assertDatabase(); + + $tmpExportFile2 = $this->exportToTemp(); + self::assertEquals( + self::writeArrayStructure(array: json_decode(file_get_contents($tmpExportFile), true)), + self::writeArrayStructure(array: json_decode(file_get_contents($tmpExportFile2), true)) + ); + } + + public static function writeArrayStructure(string $prefix = '', array $array = [], array $skipKeyList = ['id', 'boardId', 'cardId', 'stackId', 'ETag', 'permissions', 'shared']): string { + $output = ''; + $arrayIsList = array_keys($array) === range(0, count($array) - 1); + foreach ($array as $key => $value) { + $tmpPrefix = $prefix; + if (in_array($key, $skipKeyList)) { + continue; + } + if (is_array($value)) { + if ($key === 'participant' || $key === 'owner') { + $output .= $tmpPrefix . $key . ' => ' . $value['primaryKey'] . PHP_EOL; + continue; + } + $tmpPrefix .= (!$arrayIsList && !is_numeric($key) ? $key : '!!!') . ' => '; + $output .= self::writeArrayStructure($tmpPrefix, $value, $skipKeyList); + } else { + $output .= $tmpPrefix . $key . ' => ' . $value . PHP_EOL; + } + } + return $output; + } + + public function cleanDb(string $owner = 'admin'): void { + $this->connection->executeQuery('DELETE from oc_deck_boards;'); + } + + private function importFromFile(string $filePath): void { $input = $this->createMock(InputInterface::class); $input->expects($this->any()) ->method('getOption') - ->willReturnCallback(function ($arg) { + ->willReturnCallback(function ($arg) use ($filePath) { return match ($arg) { 'system' => 'DeckJson', - 'data' => __DIR__ . '/../../data/deck.json', + 'data' => $filePath, 'config' => __DIR__ . '/../../data/config-trelloJson.json', }; }); $output = $this->createMock(OutputInterface::class); - $importer = \OCP\Server::get(BoardImport::class); + $importer = self::getFreshService(BoardImport::class); $application = new Application(); $importer->setApplication($application); $importer->run($input, $output); + } - $this->assertDatabase(); - - self::overwriteService(BoardService::class, self::getFreshService(BoardService::class)); - // export to a temporary file + /** Returns the path of a deck export json */ + private function exportToTemp(): string { + \OCP\Server::get(BoardMapper::class)->flushCache(); + $application = new Application(); $input = $this->createMock(InputInterface::class); $input->expects($this->any()) ->method('getArgument') ->with('user-id') ->willReturn('admin'); $output = new BufferedOutput(); - $exporter = \OCP\Server::get(UserExport::class); + $exporter = new UserExport( + \OCP\Server::get(IAppManager::class), + self::getFreshService(BoardMapper::class), + self::getFreshService(BoardService::class), + self::getFreshService(StackMapper::class), + self::getFreshService(CardMapper::class), + self::getFreshService(AssignmentMapper::class), + ); $exporter->setApplication($application); $exporter->run($input, $output); $jsonOutput = $output->fetch(); @@ -147,46 +202,59 @@ public function testReimportOcc() { self::assertTrue(json_last_error() === JSON_ERROR_NONE); $tmpExportFile = tempnam('/tmp', 'export'); file_put_contents($tmpExportFile, $jsonOutput); + return $tmpExportFile; + } - // self::assertEquals(file_get_contents(__DIR__ . '/../../data/deck.json'), $jsonOutput); - - // cleanup test database - $this->connection->rollBack(); - $this->connection->beginTransaction(); + public function testImport() { + $importer = self::getFreshService(BoardImportService::class); + $deckJsonService = self::getFreshService(DeckJsonService::class); + $deckJsonService->setImportService($importer); - self::overwriteService(BoardService::class, self::getFreshService(BoardService::class)); - // Re-import from temporary file - $input = $this->createMock(InputInterface::class); - $input->expects($this->any()) - ->method('getOption') - ->willReturnCallback(function ($arg) use ($tmpExportFile) { - return match ($arg) { - 'system' => 'DeckJson', - 'data' => $tmpExportFile, - 'config' => __DIR__ . '/../../data/config-trelloJson.json', - }; - }); - $output = $this->createMock(OutputInterface::class); - $importer = \OCP\Server::get(BoardImport::class); - $application = new Application(); - $importer->setApplication($application); - $importer->run($input, $output); + $importer->setSystem('DeckJson'); + $importer->setImportSystem($deckJsonService); + $importer->setConfigInstance(json_decode(file_get_contents(__DIR__ . '/../../data/config-trelloJson.json'))); + $importer->setData(json_decode(file_get_contents(__DIR__ . '/../../data/deck.json'))); + $importer->import(); $this->assertDatabase(); } - public function testImport() { - $importer = \OCP\Server::get(BoardImportService::class); - $deckJsonService = \OCP\Server::get(DeckJsonService::class); + public function testImportAsOtherUser() { + $importer = self::getFreshService(BoardImportService::class); + $deckJsonService = self::getFreshService(DeckJsonService::class); $deckJsonService->setImportService($importer); $importer->setSystem('DeckJson'); $importer->setImportSystem($deckJsonService); - $importer->setConfigInstance(json_decode(file_get_contents(__DIR__ . '/../../data/config-trelloJson.json'))); + $importer->setConfigInstance((object)[ + 'owner' => self::TEST_USER1 + ]); $importer->setData(json_decode(file_get_contents(__DIR__ . '/../../data/deck.json'))); $importer->import(); - $this->assertDatabase(); + $this->assertDatabase(self::TEST_USER1); + } + + public function testImportWithRemap() { + $importer = self::getFreshService(BoardImportService::class); + $deckJsonService = self::getFreshService(DeckJsonService::class); + $deckJsonService->setImportService($importer); + + $importer->setSystem('DeckJson'); + $importer->setImportSystem($deckJsonService); + $importer->setConfigInstance((object)[ + 'owner' => self::TEST_USER1, + 'uidRelation' => (object)[ + 'alice' => self::TEST_USER2, + 'jane' => self::TEST_USER3, + ], + ]); + $importer->setData(json_decode(file_get_contents(__DIR__ . '/../../data/deck.json'))); + $importer->import(); + + $this->assertDatabase(self::TEST_USER1); + $otherUserboards = self::getFreshService(BoardMapper::class)->findAllByUser(self::TEST_USER2); + self::assertCount(1, $otherUserboards); } /** @@ -195,17 +263,21 @@ public function testImport() { * @return T */ private function getFreshService(string $className): mixed { - return \OC::$server->getRegisteredAppContainer('deck')->resolve($className); + $fresh = \OC::$server->getRegisteredAppContainer('deck')->resolve($className); + self::overwriteService($className, $fresh); + return $fresh; } - public function assertDatabase() { - $permissionService = \OCP\Server::get(PermissionService::class); - $permissionService->setUserId('admin'); - $boardMapper = \OCP\Server::get(BoardMapper::class); - $stackMapper = \OCP\Server::get(StackMapper::class); - $cardMapper = \OCP\Server::get(CardMapper::class); + public function assertDatabase(string $owner = 'admin') { + $permissionService = self::getFreshService(PermissionService::class); + $permissionService->setUserId($owner); + self::getFreshService(BoardService::class); + self::getFreshService(CardService::class); + $boardMapper = self::getFreshService(BoardMapper::class); + $stackMapper = self::getFreshService(StackMapper::class); + $cardMapper = self::getFreshService(CardMapper::class); - $boards = $boardMapper->findAllByOwner('admin'); + $boards = $boardMapper->findAllByOwner($owner); $boardNames = array_map(fn ($board) => $board->getTitle(), $boards); self::assertEquals(2, count($boards)); @@ -213,7 +285,7 @@ public function assertDatabase() { self::assertEntity(Board::fromRow([ 'title' => 'My test board', 'color' => 'e0ed31', - 'owner' => 'admin', + 'owner' => $owner, 'lastModified' => 1689667796, ]), $board); $boardService = $this->getFreshService(BoardService::class); @@ -230,16 +302,19 @@ public function assertDatabase() { 'title' => 'A', 'order' => 999, 'boardId' => $boards[0]->getId(), + 'lastModified' => 1689667779, ]), $stacks[0]); self::assertEntity(Stack::fromRow([ 'title' => 'B', 'order' => 999, 'boardId' => $boards[0]->getId(), + 'lastModified' => 1689667796, ]), $stacks[1]); self::assertEntity(Stack::fromRow([ 'title' => 'C', 'order' => 999, 'boardId' => $boards[0]->getId(), + 'lastModified' => 0, ]), $stacks[2]); $cards = $cardMapper->findAll($stacks[0]->getId()); @@ -249,7 +324,7 @@ public function assertDatabase() { 'type' => 'plain', 'lastModified' => 1689667779, 'createdAt' => 1689667569, - 'owner' => 'admin', + 'owner' => $owner, 'duedate' => new \DateTime('2050-07-24T22:00:00.000000+0000'), 'order' => 999, 'stackId' => $stacks[0]->getId(), @@ -275,10 +350,10 @@ public function assertDatabase() { self::assertEntity(Board::fromRow([ 'title' => 'Shared board', 'color' => '30b6d8', - 'owner' => 'admin', + 'owner' => $owner, ]), $sharedBoard, true); - $stackService = \OCP\Server::get(StackService::class); + $stackService = self::getFreshService(StackService::class); $stacks = $stackService->findAll($board->getId()); self::assertEntityInArray(Label::fromParams([ 'title' => 'L2', @@ -334,9 +409,8 @@ public static function assertEntity(Entity $expected, Entity $actual, bool $chec } public function tearDown(): void { - if ($this->connection->inTransaction()) { - $this->connection->rollBack(); - } + $this->cleanDb(); + $this->cleanDb(self::TEST_USER1); parent::tearDown(); } } diff --git a/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php b/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php index fb476ecc9..0f0b8aa33 100644 --- a/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php +++ b/tests/unit/Service/Importer/Systems/DeckJsonServiceTest.php @@ -65,13 +65,13 @@ public function testGetBoardWithSuccess() { $data = json_decode(file_get_contents(__DIR__ . '/../../../../data/deck.json')); $importService->setData($data); - $configInstance = json_decode(file_get_contents(__DIR__ . '/../../../../data/config-trelloJson.json')); + $configInstance = json_decode(file_get_contents(__DIR__ . '/../../../../data/config-deckJson.json')); $importService->setConfigInstance($configInstance); $owner = $this->createMock(IUser::class); $owner ->method('getUID') - ->willReturn('owner'); + ->willReturn('admin'); $importService->setConfig('owner', $owner); $this->service->setImportService($importService); From 3039d6dce20f49e4bc9f22088286c577ffd1bf81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 11 Aug 2023 18:09:25 +0200 Subject: [PATCH 17/26] fix: Add output for individual failures or skipped parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Db/Assignment.php | 13 ++ .../Importer/BoardImportCommandService.php | 69 ++++++---- lib/Service/Importer/BoardImportService.php | 120 +++++++++++++----- tests/stub.phpstub | 2 + .../Importer/BoardImportServiceTest.php | 7 +- 5 files changed, 157 insertions(+), 54 deletions(-) diff --git a/lib/Db/Assignment.php b/lib/Db/Assignment.php index 56207d36a..282581d58 100644 --- a/lib/Db/Assignment.php +++ b/lib/Db/Assignment.php @@ -41,4 +41,17 @@ public function __construct() { $this->addType('type', 'integer'); $this->addResolvable('participant'); } + + public function getTypeString(): string { + switch ($this->getType()) { + case self::TYPE_USER: + return 'user'; + case self::TYPE_GROUP: + return 'group'; + case self::TYPE_CIRCLE: + return 'circle'; + } + + return 'unknown'; + } } diff --git a/lib/Service/Importer/BoardImportCommandService.php b/lib/Service/Importer/BoardImportCommandService.php index 23257e499..64d57622c 100644 --- a/lib/Service/Importer/BoardImportCommandService.php +++ b/lib/Service/Importer/BoardImportCommandService.php @@ -25,7 +25,6 @@ use OCA\Deck\Exceptions\ConflictException; use OCA\Deck\NotFoundException; -use OCA\Deck\Service\Importer\Systems\DeckJsonService; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -78,11 +77,12 @@ public function getOutput(): OutputInterface { protected function validateConfig(): void { // FIXME: Make config optional for deck plain importer (but use a call on the importer insterad) - if ($this->getImportSystem() instanceof DeckJsonService) { - return; - } try { $config = $this->getInput()->getOption('config'); + if (!$config) { + return; + } + if (is_string($config)) { if (!is_file($config)) { throw new NotFoundException('It\'s not a valid config file.'); @@ -191,33 +191,56 @@ protected function validateData(): void { public function bootstrap(): void { $this->setSystem($this->getInput()->getOption('system')); parent::bootstrap(); + + $this->registerErrorCollector(function ($error, $exception) { + $message = $error; + if ($exception instanceof \Throwable) { + $message .= ': ' . $exception->getMessage(); + } + $this->getOutput()->writeln('' . $message . ''); + if ($exception instanceof \Throwable && $this->getOutput()->isVeryVerbose()) { + $this->getOutput()->writeln($exception->getTraceAsString()); + } + }); + + $this->registerOutputCollector(function ($info) { + if ($this->getOutput()->isVerbose()) { + $this->getOutput()->writeln('' . $info . '', ); + } + }); } public function import(): void { $this->getOutput()->writeln('Starting import...'); $this->bootstrap(); + $this->validateSystem(); + $this->validateConfig(); $boards = $this->getImportSystem()->getBoards(); foreach ($boards as $board) { - $this->reset(); - $this->setData($board); - $this->getOutput()->writeln('Importing board "' . $this->getBoard()->getTitle() . '".'); - $this->importBoard(); - $this->getOutput()->writeln('Assign users to board...'); - $this->importAcl(); - $this->getOutput()->writeln('Importing labels...'); - $this->importLabels(); - $this->getOutput()->writeln('Importing stacks...'); - $this->importStacks(); - $this->getOutput()->writeln('Importing cards...'); - $this->importCards(); - $this->getOutput()->writeln('Assign cards to labels...'); - $this->assignCardsToLabels(); - $this->getOutput()->writeln('Importing comments...'); - $this->importComments(); - $this->getOutput()->writeln('Importing participants...'); - $this->importCardAssignments(); - $this->getOutput()->writeln('Finished board import of "' . $this->getBoard()->getTitle() . '"'); + try { + $this->reset(); + $this->setData($board); + $this->getOutput()->writeln('Importing board "' . $board->title . '".'); + $this->importBoard(); + $this->getOutput()->writeln('Assign users to board...'); + $this->importAcl(); + $this->getOutput()->writeln('Importing labels...'); + $this->importLabels(); + $this->getOutput()->writeln('Importing stacks...'); + $this->importStacks(); + $this->getOutput()->writeln('Importing cards...'); + $this->importCards(); + $this->getOutput()->writeln('Assign cards to labels...'); + $this->assignCardsToLabels(); + $this->getOutput()->writeln('Importing comments...'); + $this->importComments(); + $this->getOutput()->writeln('Importing participants...'); + $this->importCardAssignments(); + $this->getOutput()->writeln('Finished board import of "' . $this->getBoard()->getTitle() . '"'); + } catch (\Exception $e) { + $this->output->writeln('Import failed for board ' . $board->title . ': ' . $e->getMessage() . ''); + } } } } diff --git a/lib/Service/Importer/BoardImportService.php b/lib/Service/Importer/BoardImportService.php index ba08c7a93..b4468b7b7 100644 --- a/lib/Service/Importer/BoardImportService.php +++ b/lib/Service/Importer/BoardImportService.php @@ -49,6 +49,7 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\IUserManager; use OCP\Server; +use Psr\Log\LoggerInterface; class BoardImportService { private string $system = ''; @@ -70,6 +71,11 @@ class BoardImportService { private $data; private Board $board; + /** @var callable[] */ + private array $errorCollectors = []; + /** @var callable[] */ + private array $outputCollectors = []; + public function __construct( private IUserManager $userManager, private BoardMapper $boardMapper, @@ -80,7 +86,8 @@ public function __construct( private AttachmentMapper $attachmentMapper, private CardMapper $cardMapper, private ICommentsManager $commentsManager, - private IEventDispatcher $eventDispatcher + private IEventDispatcher $eventDispatcher, + private LoggerInterface $logger ) { $this->board = new Board(); $this->disableCommentsEvents(); @@ -88,6 +95,28 @@ public function __construct( $this->config = new \stdClass(); } + public function registerErrorCollector(callable $errorCollector): void { + $this->errorCollectors[] = $errorCollector; + } + + public function registerOutputCollector(callable $outputCollector): void { + $this->outputCollectors[] = $outputCollector; + } + + private function addError(string $message, $exception): void { + $message .= ' (on board ' . $this->getBoard()->getTitle() . ')'; + foreach ($this->errorCollectors as $errorCollector) { + $errorCollector($message, $exception); + } + $this->logger->error($message, ['exception' => $exception]); + } + + private function addOutput(string $message): void { + foreach ($this->outputCollectors as $outputCollector) { + $outputCollector($message); + } + } + private function disableCommentsEvents(): void { if (defined('PHPUNIT_RUN')) { return; @@ -117,6 +146,7 @@ public function import(): void { $this->importComments(); $this->importCardAssignments(); } catch (\Throwable $th) { + $this->logger->error('Failed to import board', ['exception' => $th]); throw new BadRequestException($th->getMessage()); } } @@ -195,6 +225,10 @@ public function reset(): void { public function importBoard(): void { $board = $this->getImportSystem()->getBoard(); + if (!$this->userManager->userExists($board->getOwner())) { + throw new \Exception('Target owner ' . $board->getOwner() . ' not found. Please provide a mapping through the import config.'); + } + if ($board) { $this->boardMapper->insert($board); $this->board = $board; @@ -211,8 +245,13 @@ public function getBoard(bool $reset = false): Board { public function importAcl(): void { $aclList = $this->getImportSystem()->getAclList(); foreach ($aclList as $code => $acl) { - $this->aclMapper->insert($acl); - $this->getImportSystem()->updateAcl($code, $acl); + try { + $this->aclMapper->insert($acl); + $this->getImportSystem()->updateAcl($code, $acl); + } catch (\Exception $e) { + $this->addError('Failed to import acl rule for ' . $acl->getParticipant(), $e); + + } } $this->getBoard()->setAcl($aclList); } @@ -220,8 +259,12 @@ public function importAcl(): void { public function importLabels(): void { $labels = $this->getImportSystem()->getLabels(); foreach ($labels as $code => $label) { - $this->labelMapper->insert($label); - $this->getImportSystem()->updateLabel($code, $label); + try { + $this->labelMapper->insert($label); + $this->getImportSystem()->updateLabel($code, $label); + } catch (\Exception $e) { + $this->addError('Failed to import label ' . $label->getTitle(), $e); + } } $this->getBoard()->setLabels($labels); } @@ -229,8 +272,12 @@ public function importLabels(): void { public function importStacks(): void { $stacks = $this->getImportSystem()->getStacks(); foreach ($stacks as $code => $stack) { - $this->stackMapper->insert($stack); - $this->getImportSystem()->updateStack($code, $stack); + try { + $this->stackMapper->insert($stack); + $this->getImportSystem()->updateStack($code, $stack); + } catch (\Exception $e) { + $this->addError('Failed to import list ' . $stack->getTitle(), $e); + } } $this->getBoard()->setStacks(array_values($stacks)); } @@ -238,22 +285,26 @@ public function importStacks(): void { public function importCards(): void { $cards = $this->getImportSystem()->getCards(); foreach ($cards as $code => $card) { - $createdAt = $card->getCreatedAt(); - $lastModified = $card->getLastModified(); - $this->cardMapper->insert($card); - $updateDate = false; - if ($createdAt && $createdAt !== $card->getCreatedAt()) { - $card->setCreatedAt($createdAt); - $updateDate = true; - } - if ($lastModified && $lastModified !== $card->getLastModified()) { - $card->setLastModified($lastModified); - $updateDate = true; - } - if ($updateDate) { - $this->cardMapper->update($card, false); + try { + $createdAt = $card->getCreatedAt(); + $lastModified = $card->getLastModified(); + $this->cardMapper->insert($card); + $updateDate = false; + if ($createdAt && $createdAt !== $card->getCreatedAt()) { + $card->setCreatedAt($createdAt); + $updateDate = true; + } + if ($lastModified && $lastModified !== $card->getLastModified()) { + $card->setLastModified($lastModified); + $updateDate = true; + } + if ($updateDate) { + $this->cardMapper->update($card, false); + } + $this->getImportSystem()->updateCard($code, $card); + } catch (\Exception $e) { + $this->addError('Failed to import card ' . $card->getTitle(), $e); } - $this->getImportSystem()->updateCard($code, $card); } } @@ -274,11 +325,15 @@ public function assignCardsToLabels(): void { $data = $this->getImportSystem()->getCardLabelAssignment(); foreach ($data as $cardId => $assignemnt) { foreach ($assignemnt as $assignmentId => $labelId) { - $this->assignCardToLabel( - $cardId, - $labelId - ); - $this->getImportSystem()->updateCardLabelsAssignment($cardId, $assignmentId, $labelId); + try { + $this->assignCardToLabel( + $cardId, + $labelId + ); + $this->getImportSystem()->updateCardLabelsAssignment($cardId, $assignmentId, $labelId); + } catch (\Exception $e) { + $this->addError('Failed to assign label ' . $labelId . ' to ' . $cardId, $e); + } } } } @@ -320,9 +375,14 @@ private function insertComment(string $cardId, IComment $comment): void { public function importCardAssignments(): void { $allAssignments = $this->getImportSystem()->getCardAssignments(); foreach ($allAssignments as $cardId => $assignments) { - foreach ($assignments as $assignmentId => $assignment) { - $this->assignmentMapper->insert($assignment); - $this->getImportSystem()->updateCardAssignment($cardId, $assignmentId, $assignment); + foreach ($assignments as $assignment) { + try { + $assignment = $this->assignmentMapper->insert($assignment); + $this->getImportSystem()->updateCardAssignment($cardId, (string)$assignment->getId(), $assignment); + $this->addOutput('Assignment ' . $assignment->getParticipant() . ' added'); + } catch (NotFoundException $e) { + $this->addError('No origin or mapping found for card "' . $cardId . '" and ' . $assignment->getTypeString() .' assignment "' . $assignment->getParticipant(), $e); + } } } } diff --git a/tests/stub.phpstub b/tests/stub.phpstub index 156fff6ff..2adce3dd8 100644 --- a/tests/stub.phpstub +++ b/tests/stub.phpstub @@ -193,6 +193,8 @@ namespace Symfony\Component\Console\Output { class OutputInterface { public const VERBOSITY_VERBOSE = 1; public function writeln($text, int $flat = 0) {} + public function isVerbose(): bool {} + public function isVeryVerbose(): bool {} } } diff --git a/tests/unit/Service/Importer/BoardImportServiceTest.php b/tests/unit/Service/Importer/BoardImportServiceTest.php index c652599b0..d367178ac 100644 --- a/tests/unit/Service/Importer/BoardImportServiceTest.php +++ b/tests/unit/Service/Importer/BoardImportServiceTest.php @@ -43,6 +43,7 @@ use OCP\IUser; use OCP\IUserManager; use PHPUnit\Framework\MockObject\MockObject; +use Psr\Log\LoggerInterface; class BoardImportServiceTest extends \Test\TestCase { /** @var IDBConnection|MockObject */ @@ -92,7 +93,8 @@ public function setUp(): void { $this->attachmentMapper, $this->cardMapper, $this->commentsManager, - $this->eventDispatcher + $this->eventDispatcher, + $this->createMock(LoggerInterface::class), ); $this->boardImportService->setSystem('trelloJson'); @@ -145,6 +147,9 @@ public function setUp(): void { } public function testImportSuccess() { + $this->userManager->method('userExists') + ->willReturn(true); + $this->boardMapper ->expects($this->once()) ->method('insert'); From 4ee7203990814316f4be105bf10c8e42bac56c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 11 Aug 2023 18:20:08 +0200 Subject: [PATCH 18/26] fix: Map card assignments through mapping config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Service/Importer/Systems/DeckJsonService.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index d2c622154..4de3ec459 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -77,6 +77,11 @@ public function validateUsers(): void { } public function mapMember($uid): ?string { + $ownerMap = $this->mapOwner($uid); + if ($ownerMap !== $uid) { + return $ownerMap; + } + $uidCandidate = isset($this->members[$uid]) ? $this->members[$uid]?->getUID() ?? null : null; if ($uidCandidate) { return $uidCandidate; @@ -104,7 +109,7 @@ public function getCardAssignments(): array { foreach ($sourceCard->assignedUsers as $idMember) { $assignment = new Assignment(); $assignment->setCardId($this->cards[$sourceCard->id]->getId()); - $assignment->setParticipant($idMember->participant->uid); + $assignment->setParticipant($this->mapMember($idMember->participant->uid ?? $idMember->participant)); $assignment->setType($idMember->participant->type); $assignments[$sourceCard->id][] = $assignment; } From 5b81f4b614fcab70125a87c3ee0c872d80295a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 11 Aug 2023 18:23:56 +0200 Subject: [PATCH 19/26] chore: Cleanup some outdated fixme comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Service/Importer/BoardImportCommandService.php | 1 - lib/Service/Importer/Systems/DeckJsonService.php | 3 --- 2 files changed, 4 deletions(-) diff --git a/lib/Service/Importer/BoardImportCommandService.php b/lib/Service/Importer/BoardImportCommandService.php index 64d57622c..7be2877d8 100644 --- a/lib/Service/Importer/BoardImportCommandService.php +++ b/lib/Service/Importer/BoardImportCommandService.php @@ -76,7 +76,6 @@ public function getOutput(): OutputInterface { } protected function validateConfig(): void { - // FIXME: Make config optional for deck plain importer (but use a call on the importer insterad) try { $config = $this->getInput()->getOption('config'); if (!$config) { diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index 4de3ec459..4e4af4144 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -184,7 +184,6 @@ public function getStacks(): array { $card->stackId = $index; $this->tmpCards[] = $card; } - // TODO: check older exports as currently there is a bug that adds lists to it with different index } } return $return; @@ -221,7 +220,6 @@ public function getAclList(): array { $board = $this->getImportService()->getData(); $return = []; foreach ($board->acl as $aclData) { - // FIXME: Figure out mapping $acl = new Acl(); $acl->setBoardId($this->getImportService()->getBoard()->getId()); $acl->setType($aclData->type); @@ -236,7 +234,6 @@ public function getAclList(): array { if ($participant) { $return[] = $acl; } - // TODO: Once we have error collection we should catch non-existing users } return $return; } From 3a5d5bd449933feb367d23ed967c2fc85fb146fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 11 Aug 2023 18:47:38 +0200 Subject: [PATCH 20/26] fix: Only map owner for user mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Service/Importer/Systems/DeckJsonService.php | 2 +- tests/integration/database/TransferOwnershipTest.php | 2 +- tests/integration/import/ImportExportTest.php | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index 4e4af4144..0fcef4ba0 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -78,7 +78,7 @@ public function validateUsers(): void { public function mapMember($uid): ?string { $ownerMap = $this->mapOwner($uid); - if ($ownerMap !== $uid) { + if ($uid === $this->getImportService()->getData()->owner && $ownerMap !== $this->getImportService()->getData()->owner) { return $ownerMap; } diff --git a/tests/integration/database/TransferOwnershipTest.php b/tests/integration/database/TransferOwnershipTest.php index bba5e12dd..c17d8b7fc 100644 --- a/tests/integration/database/TransferOwnershipTest.php +++ b/tests/integration/database/TransferOwnershipTest.php @@ -277,7 +277,7 @@ public function testTransferSingleBoardAssignment() { // Arrange separate board next to the one being transferred $board = $this->boardService->create('Test 2', self::TEST_USER_1, '000000'); $id = $board->getId(); - $this->boardService->addAcl($id, Acl::PERMISSION_TYPE_USER, self::TEST_USER_1, true, true, true); + // $this->boardService->addAcl($id, Acl::PERMISSION_TYPE_USER, self::TEST_USER_1, true, true, true); $this->boardService->addAcl($id, Acl::PERMISSION_TYPE_GROUP, self::TEST_GROUP, true, true, true); $this->boardService->addAcl($id, Acl::PERMISSION_TYPE_USER, self::TEST_USER_3, false, true, false); $stacks[] = $this->stackService->create('Stack A', $id, 1); diff --git a/tests/integration/import/ImportExportTest.php b/tests/integration/import/ImportExportTest.php index 64c14207a..0577fc442 100644 --- a/tests/integration/import/ImportExportTest.php +++ b/tests/integration/import/ImportExportTest.php @@ -49,11 +49,11 @@ class ImportExportTest extends \Test\TestCase { private IDBConnection $connection; - private const TEST_USER1 = 'test-share-user1'; - private const TEST_USER3 = 'test-share-user3'; - private const TEST_USER2 = 'test-share-user2'; - private const TEST_USER4 = 'test-share-user4'; - private const TEST_GROUP1 = 'test-share-group1'; + private const TEST_USER1 = 'test-export-user1'; + private const TEST_USER3 = 'test-export-user3'; + private const TEST_USER2 = 'test-export-user2'; + private const TEST_USER4 = 'test-export-user4'; + private const TEST_GROUP1 = 'test-export-group1'; public static function setUpBeforeClass(): void { parent::setUpBeforeClass(); From 63eb8cf350947a401078798055b23ecc76fcead3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 11 Aug 2023 19:47:27 +0200 Subject: [PATCH 21/26] tests: ignore version of stored json for import tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- tests/integration/import/ImportExportTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/import/ImportExportTest.php b/tests/integration/import/ImportExportTest.php index 0577fc442..7b159b7e9 100644 --- a/tests/integration/import/ImportExportTest.php +++ b/tests/integration/import/ImportExportTest.php @@ -133,7 +133,7 @@ public function testReimportOcc() { ); } - public static function writeArrayStructure(string $prefix = '', array $array = [], array $skipKeyList = ['id', 'boardId', 'cardId', 'stackId', 'ETag', 'permissions', 'shared']): string { + public static function writeArrayStructure(string $prefix = '', array $array = [], array $skipKeyList = ['id', 'boardId', 'cardId', 'stackId', 'ETag', 'permissions', 'shared', 'version']): string { $output = ''; $arrayIsList = array_keys($array) === range(0, count($array) - 1); foreach ($array as $key => $value) { From 6891b80eb2bf9a194398f032df909770a4cfc11a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 11 Aug 2023 19:49:01 +0200 Subject: [PATCH 22/26] style: fix php-cs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Service/Importer/BoardImportService.php | 1 - tests/integration/import/ImportExportTest.php | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/Service/Importer/BoardImportService.php b/lib/Service/Importer/BoardImportService.php index b4468b7b7..43677fdc5 100644 --- a/lib/Service/Importer/BoardImportService.php +++ b/lib/Service/Importer/BoardImportService.php @@ -250,7 +250,6 @@ public function importAcl(): void { $this->getImportSystem()->updateAcl($code, $acl); } catch (\Exception $e) { $this->addError('Failed to import acl rule for ' . $acl->getParticipant(), $e); - } } $this->getBoard()->setAcl($aclList); diff --git a/tests/integration/import/ImportExportTest.php b/tests/integration/import/ImportExportTest.php index 7b159b7e9..297b823e7 100644 --- a/tests/integration/import/ImportExportTest.php +++ b/tests/integration/import/ImportExportTest.php @@ -47,7 +47,6 @@ * @group DB */ class ImportExportTest extends \Test\TestCase { - private IDBConnection $connection; private const TEST_USER1 = 'test-export-user1'; private const TEST_USER3 = 'test-export-user3'; From f2e1a24b6b4060f6781fadc5f1f42b0bf3dfb22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Sat, 12 Aug 2023 09:10:59 +0200 Subject: [PATCH 23/26] fix: use proper owner source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- lib/Service/Importer/Systems/DeckJsonService.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/Service/Importer/Systems/DeckJsonService.php b/lib/Service/Importer/Systems/DeckJsonService.php index 0fcef4ba0..10159b15b 100644 --- a/lib/Service/Importer/Systems/DeckJsonService.php +++ b/lib/Service/Importer/Systems/DeckJsonService.php @@ -78,7 +78,9 @@ public function validateUsers(): void { public function mapMember($uid): ?string { $ownerMap = $this->mapOwner($uid); - if ($uid === $this->getImportService()->getData()->owner && $ownerMap !== $this->getImportService()->getData()->owner) { + $sourceId = ($this->getImportService()->getData()->owner->primaryKey ?? $this->getImportService()->getData()->owner); + + if ($uid === $sourceId && $ownerMap !== $sourceId) { return $ownerMap; } From 0bdba679534836c780024962525ada5ef3984355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Sat, 12 Aug 2023 09:14:31 +0200 Subject: [PATCH 24/26] ci(cypress): Catch resize observer loop limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- cypress/support/e2e.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cypress/support/e2e.js b/cypress/support/e2e.js index b6ca34662..358fb9e21 100644 --- a/cypress/support/e2e.js +++ b/cypress/support/e2e.js @@ -16,5 +16,9 @@ // Import commands.js using ES2015 syntax: import './commands.js' +Cypress.on('uncaught:exception', (err) => { + return !err.message.includes('ResizeObserver loop limit exceeded'), +}) + // Alternatively you can use CommonJS syntax: // require('./commands') From a352aaf924f9db7fcbe24cb75a767102260227cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Sat, 12 Aug 2023 09:16:34 +0200 Subject: [PATCH 25/26] ci(integration): Fix query counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- .github/workflows/integration.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a84b5f567..bd3d09d9d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -71,7 +71,7 @@ jobs: path: apps/${{ env.APP_NAME }} - name: Set up php ${{ matrix.php-versions }} - uses: shivammathur/setup-php@2.24.0 + uses: shivammathur/setup-php@2.25.5 with: php-version: ${{ matrix.php-versions }} extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_sqlite, mysql, pdo_mysql, pgsql, pdo_pgsql, apcu @@ -98,7 +98,7 @@ jobs: cat config/config.php ./occ user:list ./occ app:enable --force ${{ env.APP_NAME }} - ./occ config:system:set query_log_file --value '/home/runner/work/${{ env.APP_NAME }}/${{ env.APP_NAME }}/query.log' + ./occ config:system:set query_log_file --value "$PWD/query.log" php -S localhost:8080 & - name: Run behat @@ -123,12 +123,12 @@ jobs: myError += data.toString() } } - await exec.exec(`/bin/bash -c "cat /home/runner/work/${{ env.APP_NAME }}/${{ env.APP_NAME }}/query.log | wc -l"`, [], options) + await exec.exec(`/bin/bash -c "cat query.log | wc -l"`, [], options) msg = myOutput const queryCount = parseInt(myOutput, 10) myOutput = '' - await exec.exec('cat', ['/home/runner/work/${{ env.APP_NAME }}/${{ env.APP_NAME }}/apps/${{ env.APP_NAME }}/tests/integration/base-query-count.txt'], options) + await exec.exec('cat', ['apps/${{ env.APP_NAME }}/tests/integration/base-query-count.txt'], options) const baseCount = parseInt(myOutput, 10) const absoluteIncrease = queryCount - baseCount From c081102c13eb8caf3c08674d564f912a6f94928c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Sat, 12 Aug 2023 09:27:08 +0200 Subject: [PATCH 26/26] ci(cypress): Catch resize observer loop limit (2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- cypress/support/e2e.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypress/support/e2e.js b/cypress/support/e2e.js index 358fb9e21..07348c5c9 100644 --- a/cypress/support/e2e.js +++ b/cypress/support/e2e.js @@ -17,7 +17,7 @@ import './commands.js' Cypress.on('uncaught:exception', (err) => { - return !err.message.includes('ResizeObserver loop limit exceeded'), + return !err.message.includes('ResizeObserver loop limit exceeded') }) // Alternatively you can use CommonJS syntax: