feat: add admin level settings to configure the log handlers enabled …

…for all flows
catalyst · Jul 21, 2023 · 225ae7b · 225ae7b
1 parent 323102d
commit 225ae7b
Show file tree

Hide file tree

Showing 7 changed files with 170 additions and 57 deletions.
diff --git a/classes/admin/admin_setting_configmultiselect.php b/classes/admin/admin_setting_configmultiselect.php
@@ -0,0 +1,31 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\admin;
+
+use tool_dataflows\admin\readonly_trait;
+
+/**
+ * A custom class applying extra readonly checks to the base implementation.
+ *
+ * @package   tool_dataflows
+ * @author    Kevin Pham <kevinpham@catalyst-au.net>
+ * @copyright Catalyst IT, 2023
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class admin_setting_configmultiselect extends \admin_setting_configmultiselect {
+    use readonly_trait;
+}
diff --git a/classes/local/execution/engine.php b/classes/local/execution/engine.php
@@ -16,6 +16,7 @@
 
 namespace tool_dataflows\local\execution;
 
+use Monolog\Formatter\LineFormatter;
 use Monolog\Handler\BrowserConsoleHandler;
 use Monolog\Handler\RotatingFileHandler;
 use Monolog\Handler\StreamHandler;
@@ -24,6 +25,7 @@
 use tool_dataflows\dataflow;
 use tool_dataflows\exportable;
 use tool_dataflows\helper;
+use tool_dataflows\local\execution\logging\log_handler;
 use tool_dataflows\local\execution\logging\mtrace_handler;
 use tool_dataflows\local\service\step_service;
 use tool_dataflows\local\step\flow_cap;
@@ -148,66 +150,19 @@ class engine {
      * @param bool $automated Execution of this run was an automated trigger.
      */
     public function __construct(dataflow $dataflow, bool $isdryrun = false, $automated = true) {
-        global $CFG;
         $this->dataflow = $dataflow;
         $this->isdryrun = $isdryrun;
         $this->automated = $automated;
         $status = self::STATUS_NEW;
 
-        // Initalise a new run (only for non-dry runs). This should only be
-        // created when the engine is executed.
-        $channel = 'dataflow/' . $this->dataflow->id;
         if (!$this->isdryrun) {
             $this->run = new run;
             $this->run->dataflowid = $this->dataflow->id;
             $this->run->initialise($status, $this->export());
-            $channel .= '/' . $this->run->name;
         }
 
-        // Each channel represents a specific way of writing log information.
-        $log = new Logger($channel);
-
-        // Add a custom formatter.
-        $log->pushProcessor(new PsrLogMessageProcessor(null, true));
-
-        // Ensure step names are used if supplied.
-        $log->pushProcessor(function ($record) {
-            if (isset($this->currentstep)) {
-                $record['context']['step'] = $this->currentstep->name;
-            }
-
-            if (isset($record['context']['step'])) {
-                $record['message'] = '{step}: ' . $record['message'];
-            }
-            if (isset($record['context']['record'])) {
-                $record['message'] .= ' {record}';
-                $record['context']['record'] = json_encode($record['context']['record']);
-            }
-            return $record;
-        });
-
-        // Default Moodle handler. Always on.
-        $log->pushHandler(new mtrace_handler(Logger::INFO));
-
-        // Log to the browser's dev console for a manual run.
-        $log->pushHandler(new BrowserConsoleHandler(Logger::DEBUG));
-
-        // Dataflow run logger.
-        // $dataflowlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
-        //     'tool_dataflows' . DIRECTORY_SEPARATOR .
-        //     $this->dataflow->id . DIRECTORY_SEPARATOR . $this->run->name
-        //     . '.log';
-        // $max = 5;
-        // $log->pushHandler(new RotatingFileHandler($dataflowlogpath, $max, Logger::DEBUG));
-
-        // General dataflow logger.
-        $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
-            'tool_dataflows' . DIRECTORY_SEPARATOR .
-            $this->dataflow->id . DIRECTORY_SEPARATOR . 'flow.log';
-
-        $log->pushHandler(new StreamHandler($dataflowrunlogpath, Logger::DEBUG));
-
-        $this->logger = $log;
+        // Set up logging.
+        $this->setup_logging();
 
         // Force the dataflow to create a fresh set of variables.
         $dataflow->clear_variables();
@@ -680,7 +635,7 @@ public function set_status(int $status) {
         $level = Logger::INFO;
         if (in_array($status, self::STATUS_TERMINATORS, true)) {
             $level = Logger::NOTICE;
-            $context['data'] = $this->get_export_data();
+            $context['export'] = $this->get_export_data();
         }
         $this->logger->log($level, "Engine: dataflow '{status}'", $context);
     }
@@ -758,4 +713,86 @@ public function create_temporary_file($prefix = '____') {
     public function set_current_step(?engine_step $step) {
         $this->currentstep = $step;
     }
+
+    /**
+     * Set up the logging for this engine.
+     *
+     * This will enable any adapters / handlers enabled from the dataflow configuration.
+     * The preference for applying the rules will be from most specific to more
+     * general settings.
+     */
+    private function setup_logging() {
+        global $CFG;
+        // Initalise a new run (only for non-dry runs). This should only be
+        // created when the engine is executed.
+        $channel = 'dataflow/' . $this->dataflow->id;
+        if (isset($this->run->name)) {
+            $channel .= '/' . $this->run->name;
+        }
+
+        // Each channel represents a specific way of writing log information.
+        $log = new Logger($channel);
+
+        // Add a custom formatter.
+        $log->pushProcessor(new PsrLogMessageProcessor(null, true));
+
+        // Ensure step names are used if supplied.
+        $log->pushProcessor(function ($record) {
+            if (isset($this->currentstep)) {
+                $record['context']['step'] = $this->currentstep->name;
+            }
+
+            if (isset($record['context']['step'])) {
+                $record['message'] = '{step}: ' . $record['message'];
+            }
+            if (isset($record['context']['record'])) {
+                $record['message'] .= ' {record}';
+                $record['context']['record'] = json_encode($record['context']['record']);
+            }
+            return $record;
+        });
+
+        // Tweak the default datetime output to include microseconds.
+        $lineformatter = new LineFormatter(null, 'Y-m-d H:i:s.u');
+
+        // Log handlers.
+        $loghandlers = array_flip(explode(',', get_config('tool_dataflows', 'log_handlers')));
+
+        // Default Moodle handler. Always on.
+        $mtracehandler = new mtrace_handler(Logger::DEBUG);
+        $mtracehandler->setFormatter($lineformatter);
+        $log->pushHandler($mtracehandler);
+
+        // Log to the browser's dev console for a manual run.
+        if (isset($loghandlers[log_handler::BROWSER_CONSOLE])) {
+            $log->pushHandler(new BrowserConsoleHandler(Logger::DEBUG));
+        }
+
+        // Dataflow run logger.
+        // e.g. '[dataroot]/tool_dataflows/3/21.log' as the path.
+        if (isset($loghandlers[log_handler::FILE_PER_RUN])) {
+            $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
+                'tool_dataflows' . DIRECTORY_SEPARATOR .
+                $this->dataflow->id . DIRECTORY_SEPARATOR . $this->run->name
+                . '.log';
+
+            $streamhandler = new StreamHandler($dataflowrunlogpath, Logger::DEBUG);
+            $streamhandler->setFormatter($lineformatter);
+            $log->pushHandler($streamhandler);
+        }
+
+        // General dataflow logger (rotates daily to prevent big single log file).
+        // e.g. '[dataroot]/tool_dataflows/3-2006-01-02.log' as the path.
+        if (isset($loghandlers[log_handler::FILE_PER_DATAFLOW])) {
+            $dataflowlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
+                'tool_dataflows' . DIRECTORY_SEPARATOR .
+                $this->dataflow->id . '.log';
+
+            $rotatingfilehandler = new RotatingFileHandler($dataflowlogpath, 0, Logger::DEBUG);
+            $rotatingfilehandler->setFormatter($lineformatter);
+            $log->pushHandler($rotatingfilehandler);
+        }
+
+        $this->logger = $log;
+    }
 }
diff --git a/classes/local/execution/logging/log_handler.php b/classes/local/execution/logging/log_handler.php
@@ -0,0 +1,29 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\execution\logging;
+
+class log_handler {
+
+    /** @var string */
+    const BROWSER_CONSOLE = 'browser_console';
+
+    /** @var string */
+    const FILE_PER_DATAFLOW = 'file_per_dataflow';
+
+    /** @var string */
+    const FILE_PER_RUN = 'file_per_run';
+}
diff --git a/classes/local/execution/logging/mtrace_handler.php b/classes/local/execution/logging/mtrace_handler.php
@@ -35,7 +35,8 @@ class mtrace_handler extends AbstractHandler {
      **/
     public function handle(array $record) {
         if ($this->isHandling($record)) {
-            mtrace($record['message']);
+            $record['formatted'] = trim($this->getFormatter()->format($record));
+            mtrace($record['formatted']);
         }
     }
 }
diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php
@@ -43,6 +43,11 @@
 $string['gpg_exec_path_desc'] = 'Path to GPG executable';
 $string['gpg_key_dir'] = 'Path to keyring directory';
 $string['gpg_key_dir_desc'] = 'Path to keyring directory';
+$string['log_handlers'] = 'Log handlers';
+$string['log_handlers_desc'] = 'Additional log handlers to output dataflow logs to more destinations. The handler for mtrace is always active and cannot be disabled.';
+$string['log_handler_file_per_dataflow'] = 'File per dataflow - [dataroot]/tool_dataflows/<id>-Y-m-d.log';
+$string['log_handler_file_per_run'] = 'File per run - [dataroot]/tool_dataflows/<dataflowid>/<id>.log';
+$string['log_handler_browser_console'] = 'Browser Console';
 $string['permitted_dirs'] = 'Permitted directories';
 $string['permitted_dirs_desc'] = "List directories here to allow them to be read from/written to by dataflow steps.
     One directory per line. Each directory must be an absolute path. You can use the place holder '{\$a}' for the

diff --git a/run.php b/run.php
@@ -38,12 +38,7 @@
  * @param string $eol End of line character
  */
 function tool_dataflows_mtrace_wrapper($message, $eol) {
-    $time = microtime(true);
-    $parts = sscanf($time, '%d%f');
-    $display = userdate($time, '%Y %b %e, %H:%M:%S');
-    $micro = substr(sprintf('%0.3f', $parts[1]), 1);
     $class = '';
-
     $message = str_replace("\n", "\n    ", $message);
 
     // Mark up errors..
@@ -52,7 +47,7 @@ function tool_dataflows_mtrace_wrapper($message, $eol) {
     } else if (preg_match('/warn/im', $message)) {
         $class = 'bg-warning';
     }
-    echo html_writer::tag('div', sprintf('%s%s %s %s', $display, $micro, s($message), $eol), ['class' => $class]);
+    echo html_writer::tag('div', sprintf('%s %s', s($message), $eol), ['class' => $class]);
 }
 
 // Allow execution of single dataflow. This requires login and has different rules.

diff --git a/settings.php b/settings.php
@@ -30,6 +30,8 @@
 use tool_dataflows\admin\admin_setting_configcheckbox;
 use tool_dataflows\admin\admin_setting_configtext;
 use tool_dataflows\admin\admin_setting_configexecutable;
+use tool_dataflows\admin\admin_setting_configmultiselect;
+use tool_dataflows\local\execution\logging\log_handler;
 use tool_dataflows\manager;
 
 defined('MOODLE_INTERNAL') || die();
@@ -104,6 +106,19 @@
             PARAM_RAW
         ));
 
+        // Multi-select element used to enable different logging handlers.
+        $settings->add(new admin_setting_configmultiselect(
+            'tool_dataflows/log_handlers',
+            get_string('log_handlers', 'tool_dataflows'),
+            get_string('log_handlers_desc', 'tool_dataflows'),
+            [],
+            [
+                log_handler::BROWSER_CONSOLE => get_string('log_handler_browser_console', 'tool_dataflows'),
+                log_handler::FILE_PER_DATAFLOW => get_string('log_handler_file_per_dataflow', 'tool_dataflows'),
+                log_handler::FILE_PER_RUN => get_string('log_handler_file_per_run', 'tool_dataflows'),
+            ]
+        ));
+
         $settings->add(
             new admin_setting_configexecutable(
                 'tool_dataflows/gpg_exec_path',