fix!: adjust filename of log files to include datetime

- Add step in upgrade to update existing log files

classes/local/execution/engine.php

@@ -746,6 +746,12 @@ private function setup_logging() {
             $channel .= '/' . $this->run->name;
         }
 
+        // Set the starting time as 'now'.
+        $now = microtime(true);
+        [, $decimal] = explode('.', $now);
+        $decimal = substr($decimal, 0, 3); // Only use the first 3 digits after the decimal point.
+        $rundateformat = date("Ymd_His$decimal", $now);
+
         // Each channel represents a specific way of writing log information.
         $log = new Logger($channel);
 
@@ -785,26 +791,32 @@ private function setup_logging() {
         }
 
         // Dataflow run logger.
-        // e.g. '[dataroot]/tool_dataflows/3/21.log' as the path.
+        // Type: FILE_PER_RUN
+        // e.g. '[dataroot]/tool_dataflows/3/20060102150405-21.log' as the path.
         if (isset($loghandlers[log_handler::FILE_PER_RUN])) {
             $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
                 'tool_dataflows' . DIRECTORY_SEPARATOR .
-                $this->dataflow->id . DIRECTORY_SEPARATOR . $this->run->name
-                . '.log';
+                $this->dataflow->id . DIRECTORY_SEPARATOR .
+                $rundateformat . '_' . $this->run->name . '.log';
 
             $streamhandler = new StreamHandler($dataflowrunlogpath, Logger::DEBUG);
             $streamhandler->setFormatter($lineformatter);
             $log->pushHandler($streamhandler);
         }
 
         // General dataflow logger (rotates daily to prevent big single log file).
-        // e.g. '[dataroot]/tool_dataflows/3-2006-01-02.log' as the path.
+        // Type: FILE_PER_DATAFLOW
+        // e.g. '[dataroot]/tool_dataflows/20060102-3.log' as the path.
         if (isset($loghandlers[log_handler::FILE_PER_DATAFLOW])) {
             $dataflowlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
                 'tool_dataflows' . DIRECTORY_SEPARATOR .
                 $this->dataflow->id . '.log';
 
             $rotatingfilehandler = new RotatingFileHandler($dataflowlogpath, 0, Logger::DEBUG);
+            $dateformat = 'Ymd';
+            $filenameformat = '{date}_{filename}';
+            $rotatingfilehandler->setFilenameFormat($filenameformat, $dateformat);
+
             $rotatingfilehandler->setFormatter($lineformatter);
             $log->pushHandler($rotatingfilehandler);
         }

classes/local/step/flow_transformer_regex.php

@@ -62,6 +62,7 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) {
             get_string('flow_transformer_regex:pattern', 'tool_dataflows'),
             [
                 'placeholder' => "/[abc]/",
+                'size' => '60',
             ]
         );
         $mform->addElement(

db/upgrade.php

@@ -287,5 +287,49 @@ function xmldb_tool_dataflows_upgrade($oldversion) {
         upgrade_plugin_savepoint(true, 2023072100, 'tool', 'dataflows');
     }
 
+    // Move log files that exist across to new format. Breaking change if any
+    // dataflows implement logic based on these files based on filename format.
+    if ($oldversion < 2023110901) {
+        $path = '*.log';
+        $pattern = '/(\d+)-(\d{4})-(\d{2})-(\d{2})/m';
+        $replace = '$2$3$4_$1';
+        xmldb_tool_dataflows_logfile_rename_helper($path, $pattern, $replace);
+
+        $path = '*/*.log';
+        $pattern = '/(\d+)\/(\d+)(_)*.*\.log/m';
+        $replace = '$1/{modifiedtime}_$2.log';
+        xmldb_tool_dataflows_logfile_rename_helper($path, $pattern, $replace, true);
+
+        // Dataflows savepoint reached.
+        upgrade_plugin_savepoint(true, 2023110901, 'tool', 'dataflows');
+    }
+
     return true;
 }
+
+/**
+ * Log file helper function
+ *
+ * @param string $path
+ * @param string $pattern
+ * @param string $replace
+ * @param bool $modifiedtimeprefix whether or not to add a datetime prefix to the new log file
+ * @return bool result
+ */
+function xmldb_tool_dataflows_logfile_rename_helper(string $path, string $pattern, string $replace, $modifiedtimeprefix = false) {
+    global $CFG;
+
+    $plugindatadir = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows';
+    $files = glob($plugindatadir . DIRECTORY_SEPARATOR . $path);
+    foreach ($files as $file) {
+        $strreplace = $replace;
+        if ($modifiedtimeprefix) {
+            $newprefix = date('Ymd_His000', filemtime($file));
+            $strreplace = str_replace('{modifiedtime}', $newprefix, $replace);
+        }
+        $newlocation = preg_replace($pattern, $strreplace, $file);
+        if ($newlocation) {
+            rename($file, $newlocation);
+        }
+    }
+}

lang/en/tool_dataflows.php

@@ -47,8 +47,8 @@
 $string['gpg_key_dir_desc'] = 'Path to keyring directory';
 $string['log_handlers'] = 'Log handlers';
 $string['log_handlers_desc'] = 'Additional log handlers to output dataflow logs to more destinations. The handler for mtrace is always active and cannot be disabled. Applying the settings at the dataflow level will override settings applied at the site admin level.';
-$string['log_handler_file_per_dataflow'] = 'File per dataflow - [dataroot]/tool_dataflows/{id}-Y-m-d.log';
-$string['log_handler_file_per_run'] = 'File per run - [dataroot]/tool_dataflows/{dataflowid}/{id}.log';
+$string['log_handler_file_per_dataflow'] = 'File per dataflow - [dataroot]/tool_dataflows/Ymd_{dataflowid}.log';
+$string['log_handler_file_per_run'] = 'File per run - [dataroot]/tool_dataflows/{dataflowid}/Ymd_his_{runid}.log';
 $string['log_handler_browser_console'] = 'Browser Console';
 $string['permitted_dirs'] = 'Permitted directories';
 $string['permitted_dirs_desc'] = "List directories here to allow them to be read from/written to by dataflow steps.

version.php

@@ -25,8 +25,8 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version = 2023110900;
-$plugin->release = 2023110900;
+$plugin->version = 2023110901;
+$plugin->release = 2023110901;
 $plugin->requires = 2022112800;    // Our lowest supported Moodle (3.3.0).
 // TODO $plugin->incompatible = ;  // Available as of Moodle 3.9.0 or later.
 $plugin->component = 'tool_dataflows';