Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add periodic logging every 100th file iterated #228

Merged
merged 8 commits into from
Apr 18, 2019
62 changes: 57 additions & 5 deletions src/FileMigrationHelper.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@

namespace SilverStripe\Assets;

use Psr\Log\LoggerInterface;
use SilverStripe\Assets\Flysystem\FlysystemAssetStore;
use SilverStripe\Assets\Storage\AssetStore;
use SilverStripe\Core\Config\Config;
use SilverStripe\Core\Config\Configurable;
use SilverStripe\Core\Environment;
use SilverStripe\Core\Injector\Injectable;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\DataQuery;
Expand All @@ -34,17 +36,42 @@ class FileMigrationHelper
*/
private static $delete_invalid_files = true;

/**
* Specifies the interval for every Nth file looped at which output will be logged.
*
* @config
* @var int
*/
private static $log_interval = 100;

private static $dependencies = [
bergice marked this conversation as resolved.
Show resolved Hide resolved
'logger' => '%$' . LoggerInterface::class,
];

/** @var LoggerInterface|null */
private $logger;
bergice marked this conversation as resolved.
Show resolved Hide resolved

/**
* @param LoggerInterface $logger
*/
public function setLogger(LoggerInterface $logger)
{
$this->logger = $logger;
}

/**
* Perform migration
*
* @param string $base Absolute base path (parent of assets folder). Will default to PUBLIC_PATH
* @return int Number of files successfully migrated
* @throws \Exception
*/
public function run($base = null)
{
if (empty($base)) {
$base = PUBLIC_PATH;
}

// Check if the File dataobject has a "Filename" field.
// If not, cannot migrate
/** @skipUpgrade */
Expand All @@ -57,26 +84,36 @@ public function run($base = null)
Environment::increaseMemoryLimitTo();

// Loop over all files
$count = 0;
$processedCount = $migratedCount = 0;
$originalState = null;
if (class_exists(Versioned::class)) {
$originalState = Versioned::get_reading_mode();
Versioned::set_stage(Versioned::DRAFT);
}

foreach ($this->getFileQuery() as $file) {
$query = $this->getFileQuery();
$total = $query->count();
foreach ($query as $file) {
// Bypass the accessor and the filename from the column
$filename = $file->getField('Filename');

$success = $this->migrateFile($base, $file, $filename);

if ($processedCount % $this->config()->get('log_interval') === 0) {
if ($this->logger) {
$this->logger->info("Iterated $processedCount out of $total files. Migrated $migratedCount files.");
}
}

$processedCount++;
if ($success) {
$count++;
$migratedCount++;
}
bergice marked this conversation as resolved.
Show resolved Hide resolved
}
if (class_exists(Versioned::class)) {
Versioned::set_reading_mode($originalState);
}
return $count;
return $migratedCount;
}

/**
Expand All @@ -86,6 +123,7 @@ public function run($base = null)
* @param File $file
* @param string $legacyFilename
* @return bool True if this file is imported successfully
* @throws \SilverStripe\ORM\ValidationException
*/
protected function migrateFile($base, File $file, $legacyFilename)
{
Expand All @@ -94,6 +132,9 @@ protected function migrateFile($base, File $file, $legacyFilename)
// Make sure this legacy file actually exists
$path = $base . '/' . $legacyFilename;
if (!file_exists($path)) {
if ($this->logger) {
$this->logger->warning("$legacyFilename not migrated because the file does not exist ($path)");
}
return false;
}

Expand All @@ -104,6 +145,9 @@ protected function migrateFile($base, File $file, $legacyFilename)
if ($this->config()->get('delete_invalid_files')) {
$file->delete();
}
if ($this->logger) {
$this->logger->warning("$legacyFilename not migrated because the extension $extension is not a valid extension");
}
return false;
}

Expand Down Expand Up @@ -151,7 +195,13 @@ protected function migrateFile($base, File $file, $legacyFilename)

if (!$useLegacyFilenames) {
// removing the legacy file since it has been migrated now and not using legacy filenames
return unlink($path);
$removed = unlink($path);
if (!$removed) {
if ($this->logger) {
$this->logger->warning("$legacyFilename was migrated, but failed to remove the legacy file ($path)");
}
}
return $removed;
}
return true;
}
Expand All @@ -173,6 +223,7 @@ protected function validateFileClassname($file, $extension)
* Get list of File dataobjects to import
*
* @return DataList
* @throws \Exception
*/
protected function getFileQuery()
{
Expand All @@ -197,6 +248,7 @@ protected function getFileQuery()
*
* @deprecated 4.4.0
* @return array
* @throws \Exception
*/
protected function getFilenameArray()
{
Expand Down