Merge pull request #40 from luyadev/keywords-to-content

add keywords
luyadev · Aug 10, 2021 · 3462307 · 3462307
2 parents 68fc0ca + 4791823
commit 3462307
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 15 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).
 In order to read more about upgrading and BC breaks have a look at the [UPGRADE Document](UPGRADE.md).
 
+## 3.3.0 (10. August 2021)
+
++ [#40](https://github.com/luyadev/luya-module-crawler/pull/40) Add keywords to content string in order to make them searchable.
+
 ## 3.2.4 (15. April 2021)
 
 + Adjusted the default url rule for the crawler, the action was missing before `crawler/default` now `crawler/default/index`.

diff --git a/src/crawler/ResultHandler.php b/src/crawler/ResultHandler.php
@@ -35,6 +35,14 @@ public function __construct(CrawlController $controller)
     {
         $this->controller = $controller;
     }
+
+    /**
+    * {@inheritDoc}
+    */
+    public function onSetup(Crawler $crawler)
+    {
+        Builderindex::deleteAll();
+    }
 
     /**
      * {@inheritDoc}
@@ -50,31 +58,28 @@ public function afterRun(Result $result)
             $index->url = $url;
         }
 
-        $index->content = $result->content;
+        $content = $result->content;
+        if (!empty($result->keywords)) {
+            $content.= ' ' . $result->keywords;
+        }
+
+        $index->content = $content;
         $index->title = $result->title;
         $index->description = $result->description;
         $index->language_info = $result->language;
         $index->url_found_on_page = $result->refererUrl->getNormalized();
         $index->group = $result->group;
 
         $index->save();
-        unset($index);
+        unset($index, $content);
 
         if ($this->controller->linkcheck) {
-            foreach ($result->parserResult->links as $url => $value) {
-                Link::add($url, $value, $url);
+            foreach ($result->parserResult->links as $linkUrl => $value) {
+                Link::add($linkUrl, $value, $url);
             }
         }
     }
 
-    /**
-    * {@inheritDoc}
-    */
-    public function onSetup(Crawler $crawler)
-    {
-        Builderindex::deleteAll();
-    }
-
     /**
     * {@inheritDoc}
     */

diff --git a/tests/frontend/commands/CrawlControllerTest.php b/tests/frontend/commands/CrawlControllerTest.php
@@ -20,7 +20,7 @@ public function testIndexerInterface()
     {
         new NgRestModelFixture(['modelClass' => Link::class]);
         $module = new Module('frontendcrawler');
-        $module->baseUrl = 'https://luya.io';
+        $module->baseUrl = 'https://example.com/';
         $module->indexer = [
             MyTestIndexer::class,
         ];
@@ -36,8 +36,8 @@ public function testIndexerInterface()
         $ctrl->interactive = 0;
         $ctrl->runAction('index');
 
-        $this->assertSame('5', Builderindex::find()->asArray()->count());
-        $this->assertSame('5', Index::find()->asArray()->count());
+        $this->assertSame('1', Builderindex::find()->asArray()->count());
+        $this->assertSame('1', Index::find()->asArray()->count());
     }
 }