Merge pull request #46 from luyadev/purging

purging

CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).
 In order to read more about upgrading and BC breaks have a look at the [UPGRADE Document](UPGRADE.md).
 
+## 3.5.0 (28. April 2022)
+
++ [#46](https://github.com/luyadev/luya-module-crawler/pull/46) Prevent the crawler from purge the full index when the builder index is empty. This can be disabled with the new option `--purging=1`.
+
 ## 3.4.1 (28. April 2022)
 
 + [#45](https://github.com/luyadev/luya-module-crawler/pull/45) Use transaction to sync index table when crawler finish the process.

src/crawler/ResultHandler.php

@@ -10,6 +10,7 @@
 use Nadar\Crawler\Interfaces\HandlerInterface;
 use Nadar\Crawler\Result;
 use Yii;
+use yii\console\Exception;
 use yii\helpers\Console;
 
 /**
@@ -89,8 +90,13 @@ public function onEnd(Crawler $crawler)
         $transaction = Yii::$app->db->beginTransaction();
         try {
             $keepIndexIds = [];
-
+            $currentTotal = (int) Index::find()->count();
             $total = (int) Builderindex::find()->count();
+
+            if (!$this->controller->purging && ($currentTotal > 0 && $total == 0)) {
+                throw new Exception("The old index contained {$currentTotal} while the new index is empty. Possible misconfiguration or error while crawling the website. The force an empty index us --purging=1");
+            }
+
             $i = 0;
             if ($this->controller->verbose) {
                 Console::startProgress(0, $total, 'synchronize index: ', false);

src/frontend/commands/CrawlController.php

@@ -55,6 +55,15 @@ class CrawlController extends \luya\console\Command
      */
     public $concurrent = 15;
 
+    /**
+     * @var boolean If enabled, the crawler can fully purge the index. This is by default disabled to prevent the issue that when the crawler 
+     * starts to crawler but the target host is not returning content (maybe due to the fact its down or there is a firewall issue) the crawler
+     * will finish with 0 builder index entries and override a fully available index with an empty index. Therefore this ensures that: if builder index is empty and 
+     * the index is more then 0, an exception is thrown. `if ($builderIndexCount == 0 && $indexCount > 0) { Exception }`
+     * @since 3.5.0
+     */
+    public $purging = false;
+
     /**
      * {@inheritDoc}
      */
@@ -64,6 +73,7 @@ public function options($actionID)
         $options[] = 'linkcheck';
         $options[] = 'pdfs';
         $options[] = 'concurrent';
+        $options[] = 'purging';
         return $options;
     }