From a6a8945314c8f8bbb4be63eaeeda2f133e1aa15f Mon Sep 17 00:00:00 2001 From: Kaishiyoku Date: Thu, 16 Aug 2018 23:01:21 +0200 Subject: [PATCH] MAO-39 closed #39 fixed a bug where the MAL crawler raised an exception --- app/Console/Commands/FetchMalItem.php | 54 ++--- app/Http/Controllers/MangaController.php | 2 +- app/Models/MalItem.php | 2 + composer.json | 2 +- composer.lock | 192 +++++++++++++++++- ...nk_canonical_to_url_at_mal_items_table.php | 32 +++ resources/views/manga/index.blade.php | 6 +- 7 files changed, 253 insertions(+), 37 deletions(-) create mode 100644 database/migrations/2018_08_16_204821_rename_link_canonical_to_url_at_mal_items_table.php diff --git a/app/Console/Commands/FetchMalItem.php b/app/Console/Commands/FetchMalItem.php index a63173c..906523a 100644 --- a/app/Console/Commands/FetchMalItem.php +++ b/app/Console/Commands/FetchMalItem.php @@ -4,7 +4,9 @@ use App\Models\MalItem; use Illuminate\Console\Command; -use Jikan\Jikan; +use Jikan\Exception\ParserException; +use Jikan\MyAnimeList\MalClient; +use Jikan\Request\Manga\MangaRequest; class FetchMalItem extends Command { @@ -39,31 +41,37 @@ public function __construct() */ public function handle() { - $jikan = new Jikan(); - $malItem = MalItem::firstOrNew(['mal_id' => (int) $this->argument('mal_id')]); + try { + $jikan = new MalClient(); - $this->line(' #' . $malItem->mal_id); + $malItem = MalItem::firstOrNew(['mal_id' => (int) $this->argument('mal_id')]); - $data = $jikan->Manga($malItem->mal_id)->response; + $this->line(' #' . $malItem->mal_id); - $malItem->link_canonical = $data['link_canonical']; - $malItem->title = $data['title']; - $malItem->title_english = $data['title_english']; - $malItem->title_japanese = $data['title_japanese']; - $malItem->title_synonyms = $data['title_synonyms']; - $malItem->status = $data['status']; - $malItem->image_url = $data['image_url']; - $malItem->volumes = $data['volumes']; - $malItem->chapters = $data['chapters']; - $malItem->publishing = $data['publishing']; - $malItem->rank = $data['rank']; - $malItem->score = $data['score']; - $malItem->scored_by = $data['scored_by']; - $malItem->popularity = $data['popularity']; - $malItem->members = $data['members']; - $malItem->favorites = $data['favorites']; - $malItem->synopsis = $data['synopsis']; + $mangaItem = $jikan->getManga(new MangaRequest($malItem->mal_id)); - $malItem->save(); + $malItem->url = $mangaItem->getUrl(); + $malItem->title = $mangaItem->getTitle(); + $malItem->title_english = $mangaItem->getTitleEnglish(); + $malItem->title_japanese = $mangaItem->getTitleJapanese(); + $malItem->title_synonyms = implode(';', $mangaItem->getTitleSynonyms()); + $malItem->status = $mangaItem->getStatus(); + $malItem->image_url = $mangaItem->getImageUrl(); + $malItem->volumes = $mangaItem->getVolumes(); + $malItem->chapters = $mangaItem->getChapters(); + $malItem->publishing = $mangaItem->isPublishing(); + $malItem->rank = $mangaItem->getRank(); + $malItem->score = $mangaItem->getScore(); + $malItem->scored_by = $mangaItem->getScoredBy(); + $malItem->popularity = $mangaItem->getPopularity(); + $malItem->members = $mangaItem->getMembers(); + $malItem->favorites = $mangaItem->getFavorites(); + $malItem->synopsis = $mangaItem->getSynopsis(); + + $malItem->save(); + } catch (ParserException $e) { + $this->error('Could not fetch data.'); + $this->error($e); + } } } diff --git a/app/Http/Controllers/MangaController.php b/app/Http/Controllers/MangaController.php index 956b345..22ebf03 100644 --- a/app/Http/Controllers/MangaController.php +++ b/app/Http/Controllers/MangaController.php @@ -213,7 +213,7 @@ private function createMalItemIfNecessary($request) $malId = $request->get('mal_id', null); $malItem = MalItem::find($malId); - if (!$malItem || !$malItem->link_canonical) { + if (!$malItem || !$malItem->url) { Artisan::call("mal:get_item", ['mal_id' => $malId]); } } diff --git a/app/Models/MalItem.php b/app/Models/MalItem.php index 52cf726..cec075c 100644 --- a/app/Models/MalItem.php +++ b/app/Models/MalItem.php @@ -49,6 +49,8 @@ * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\MalItem whereUpdatedAt($value) * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\MalItem whereVolumes($value) * @mixin \Eloquent + * @property string|null $url + * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\MalItem whereUrl($value) */ class MalItem extends Model { diff --git a/composer.json b/composer.json index d9c169f..0adf2aa 100644 --- a/composer.json +++ b/composer.json @@ -13,7 +13,7 @@ "albertcht/invisible-recaptcha": "1.8.3", "doctrine/dbal": "2.7.2", "fideloper/proxy": "4.0.0", - "jikan-me/jikan": "1.16.4", + "jikan-me/jikan": "2.0.0-rc.1", "kaishiyoku/laravel-menu": "2.0.0", "kaishiyoku/yaml-translation": "5.6.0", "laracasts/flash": "3.0.2", diff --git a/composer.lock b/composer.lock index cea09d5..5870e15 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c38e0c6863609febb24e729d8e12059a", + "content-hash": "616f803149019c33d1415a4ee2182637", "packages": [ { "name": "albertcht/invisible-recaptcha", @@ -1012,6 +1012,61 @@ ], "time": "2018-02-23T01:58:20+00:00" }, + { + "name": "fabpot/goutte", + "version": "v3.2.3", + "source": { + "type": "git", + "url": "https://github.com/FriendsOfPHP/Goutte.git", + "reference": "3f0eaf0a40181359470651f1565b3e07e3dd31b8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/FriendsOfPHP/Goutte/zipball/3f0eaf0a40181359470651f1565b3e07e3dd31b8", + "reference": "3f0eaf0a40181359470651f1565b3e07e3dd31b8", + "shasum": "" + }, + "require": { + "guzzlehttp/guzzle": "^6.0", + "php": ">=5.5.0", + "symfony/browser-kit": "~2.1|~3.0|~4.0", + "symfony/css-selector": "~2.1|~3.0|~4.0", + "symfony/dom-crawler": "~2.1|~3.0|~4.0" + }, + "require-dev": { + "symfony/phpunit-bridge": "^3.3 || ^4" + }, + "type": "application", + "extra": { + "branch-alias": { + "dev-master": "3.2-dev" + } + }, + "autoload": { + "psr-4": { + "Goutte\\": "Goutte" + }, + "exclude-from-classmap": [ + "Goutte/Tests" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + } + ], + "description": "A simple PHP Web Scraper", + "homepage": "https://github.com/FriendsOfPHP/Goutte", + "keywords": [ + "scraper" + ], + "time": "2018-06-29T15:13:57+00:00" + }, { "name": "fideloper/proxy", "version": "4.0.0", @@ -1336,26 +1391,31 @@ }, { "name": "jikan-me/jikan", - "version": "v1.16.4", + "version": "v2.0.0-rc.1", "source": { "type": "git", "url": "https://github.com/jikan-me/jikan.git", - "reference": "fdc8e3bc51d021bf8694d3cfed986b662706ba39" + "reference": "cd459ecf84116282b63974d6629908c8de2ae9b1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/jikan-me/jikan/zipball/fdc8e3bc51d021bf8694d3cfed986b662706ba39", - "reference": "fdc8e3bc51d021bf8694d3cfed986b662706ba39", + "url": "https://api.github.com/repos/jikan-me/jikan/zipball/cd459ecf84116282b63974d6629908c8de2ae9b1", + "reference": "cd459ecf84116282b63974d6629908c8de2ae9b1", "shasum": "" }, "require": { - "guzzlehttp/guzzle": "^6.3", - "php": "^7.0" + "fabpot/goutte": "^3.2", + "php": "^7.1" }, "require-dev": { + "doctrine/collections": "^1.5", + "jakub-onderka/php-parallel-lint": "^1.0", + "jikan-me/jikan-fixtures": "dev-master", "php-vcr/php-vcr": "~1.3.2", "php-vcr/phpunit-testlistener-vcr": "^3.0", - "phpunit/phpunit": "^6.3" + "phpro/grumphp": "^0.14.1", + "phpunit/phpunit": "^6.3", + "squizlabs/php_codesniffer": "^3.3" }, "type": "library", "autoload": { @@ -1376,7 +1436,7 @@ } ], "description": "Jikan is an unofficial MyAnimeList API", - "time": "2018-08-13T18:14:34+00:00" + "time": "2018-08-05T18:01:06+00:00" }, { "name": "kaishiyoku/laravel-html-purifier", @@ -2749,6 +2809,63 @@ ], "time": "2018-07-13T07:04:35+00:00" }, + { + "name": "symfony/browser-kit", + "version": "v4.1.3", + "source": { + "type": "git", + "url": "https://github.com/symfony/browser-kit.git", + "reference": "c55fe9257003b2d95c0211b3f6941e8dfd26dffd" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/browser-kit/zipball/c55fe9257003b2d95c0211b3f6941e8dfd26dffd", + "reference": "c55fe9257003b2d95c0211b3f6941e8dfd26dffd", + "shasum": "" + }, + "require": { + "php": "^7.1.3", + "symfony/dom-crawler": "~3.4|~4.0" + }, + "require-dev": { + "symfony/css-selector": "~3.4|~4.0", + "symfony/process": "~3.4|~4.0" + }, + "suggest": { + "symfony/process": "" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.1-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Component\\BrowserKit\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony BrowserKit Component", + "homepage": "https://symfony.com", + "time": "2018-07-26T09:10:45+00:00" + }, { "name": "symfony/console", "version": "v4.1.3", @@ -2926,6 +3043,63 @@ "homepage": "https://symfony.com", "time": "2018-07-26T11:24:31+00:00" }, + { + "name": "symfony/dom-crawler", + "version": "v4.1.3", + "source": { + "type": "git", + "url": "https://github.com/symfony/dom-crawler.git", + "reference": "1c4519d257e652404c3aa550207ccd8ada66b38e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/1c4519d257e652404c3aa550207ccd8ada66b38e", + "reference": "1c4519d257e652404c3aa550207ccd8ada66b38e", + "shasum": "" + }, + "require": { + "php": "^7.1.3", + "symfony/polyfill-ctype": "~1.8", + "symfony/polyfill-mbstring": "~1.0" + }, + "require-dev": { + "symfony/css-selector": "~3.4|~4.0" + }, + "suggest": { + "symfony/css-selector": "" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.1-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Component\\DomCrawler\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony DomCrawler Component", + "homepage": "https://symfony.com", + "time": "2018-07-26T11:00:49+00:00" + }, { "name": "symfony/event-dispatcher", "version": "v4.1.3", diff --git a/database/migrations/2018_08_16_204821_rename_link_canonical_to_url_at_mal_items_table.php b/database/migrations/2018_08_16_204821_rename_link_canonical_to_url_at_mal_items_table.php new file mode 100644 index 0000000..6935e0b --- /dev/null +++ b/database/migrations/2018_08_16_204821_rename_link_canonical_to_url_at_mal_items_table.php @@ -0,0 +1,32 @@ +renameColumn('link_canonical', 'url'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('mal_items', function (Blueprint $table) { + $table->renameColumn('url', 'link_canonical'); + }); + } +} diff --git a/resources/views/manga/index.blade.php b/resources/views/manga/index.blade.php index 001bd18..0d39277 100644 --- a/resources/views/manga/index.blade.php +++ b/resources/views/manga/index.blade.php @@ -25,12 +25,12 @@
{{ $manga->name }} - @if ($manga->malItem && $manga->malItem->link_canonical) - {!! Html::decode(Html::link($manga->malItem->link_canonical, '', ['title' => __('manga.index.manga_at_myanimelist', ['name' => $manga->name])])) !!} + @if ($manga->malItem && $manga->malItem->url) + {!! Html::decode(Html::link($manga->malItem->url, '', ['title' => __('manga.index.manga_at_myanimelist', ['name' => $manga->name])])) !!} @endif
- @if ($manga->malItem && $manga->malItem->link_canonical) + @if ($manga->malItem && $manga->malItem->url)
{{ __('manga.index.mal_score') }}: {{ formatNumber($manga->malItem->score, 2) }}
@endif