From 44c72b132a7f4a168f2bc65c75aa564cc0608f80 Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Tue, 13 Dec 2011 08:19:37 +0000 Subject: [PATCH 01/14] Update processor to run only configured source plugins Grab a list of enabled plugins and iterate through, rather than grabbing a list of all plugins and attempting to match against the enabled ones. --- source/lib/DownloadDispatcher/Processor.class.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/lib/DownloadDispatcher/Processor.class.php b/source/lib/DownloadDispatcher/Processor.class.php index e7a766d..a5b7ac5 100644 --- a/source/lib/DownloadDispatcher/Processor.class.php +++ b/source/lib/DownloadDispatcher/Processor.class.php @@ -23,7 +23,7 @@ class DownloadDispatcher_Processor { $plugin = DownloadDispatcher_Sync_PluginFactory::create($plugin_name, $config, $log, $instance); $plugin->run(); - } catch(SihnonFramework_Exception_LogException $e) { + } catch(SihnonFramework_Exception_PluginException $e) { SihnonFramework_LogEntry::warning($log, $e->getMessage()); } } @@ -31,13 +31,14 @@ class DownloadDispatcher_Processor { // Find the list of available source plugins DownloadDispatcher_Source_PluginFactory::scan(); - $source_plugins = DownloadDispatcher_Source_PluginFactory::getValidPlugins(); - - $enabled_plugins = $config->get('sources'); + $source_plugins = $config->get('sources'); foreach ($source_plugins as $plugin_name) { - if (in_array($plugin_name, $enabled_plugins)) { + try { $plugin = DownloadDispatcher_Source_PluginFactory::create($plugin_name, $config, $log); $plugin->run(); + + } catch(DownloadDispatcher_Exception_PluginException $e) { + SihnonFramework_LogEntry::warning($log, $e->getMessage()); } } } From 0291ef051c1d194ab62b5646323a0576ceeb5fac Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Tue, 13 Dec 2011 08:29:18 +0000 Subject: [PATCH 02/14] Add persistent caching to Source plugin Store and query a cache file to look for previously handled source files. --- .../Source/PluginBase.class.php | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/source/lib/DownloadDispatcher/Source/PluginBase.class.php b/source/lib/DownloadDispatcher/Source/PluginBase.class.php index 72567c3..1ec4ed9 100644 --- a/source/lib/DownloadDispatcher/Source/PluginBase.class.php +++ b/source/lib/DownloadDispatcher/Source/PluginBase.class.php @@ -2,11 +2,26 @@ class DownloadDispatcher_Source_PluginBase extends DownloadDispatcher_PluginBase { - static protected $source_cache = array(); + static protected $cache; + + static protected $source_cache = null; + static protected $cache_file = 'source_cache'; + static protected $cache_lifetime = 86400; protected function init_cache() { + if ( ! static::$cache) { + static::$cache = DownloadDispatcher_Main::instance()->cache(); + } + + if (is_null(static::$source_cache)) { + try { + static::$source_cache = static::$cache->fetch(static::$cache_file, static::$cache_lifetime); + } catch (SihnonFramework_Exception_CacheObjectNotFound $e) { + static::$source_cache = array(); + } + } + if ( ! array_key_exists(get_called_class(), static::$source_cache)) { - // TODO - attempt to load data from persistent storage static::$source_cache[get_called_class()] = array(); } } @@ -18,7 +33,7 @@ class DownloadDispatcher_Source_PluginBase extends DownloadDispatcher_PluginBase static::$source_cache[get_called_class()][] = $file; } - // TODO - flush cache to persistent storage + static::$cache->store($cache_file, static::$source_cache); } protected function check_processed($file) { From 0cbe522a67cc8e81cf75874fd7532096acc52c9a Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Sat, 31 Dec 2011 01:22:09 +0000 Subject: [PATCH 03/14] Remove sample files from MediaFilesIterator --- .../Utility/MediaFilesIterator.class.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php b/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php index 1ce7df2..b5c8d62 100644 --- a/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php +++ b/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php @@ -1,8 +1,14 @@ current()->getFilename()); + public function accept() { + $filename = $this->current()->getFilename(); + if (preg_match('/^sample/', $filename)) { + return false; + } + if (preg_match('/(? Date: Sat, 31 Dec 2011 01:25:41 +0000 Subject: [PATCH 04/14] Complete TV move/rename operations --- .gitmodules | 3 + source/3rdparty/tvrenamer | 1 + .../Source/Plugin/TV.class.php | 179 ++++++++++++++++-- .../Source/PluginBase.class.php | 16 +- 4 files changed, 178 insertions(+), 21 deletions(-) create mode 100644 .gitmodules create mode 160000 source/3rdparty/tvrenamer diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0c825a5 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "source/3rdparty/tvrenamer"] + path = source/3rdparty/tvrenamer + url = https://github.com/meermanr/TVSeriesRenamer.git diff --git a/source/3rdparty/tvrenamer b/source/3rdparty/tvrenamer new file mode 160000 index 0000000..b1eb1ce --- /dev/null +++ b/source/3rdparty/tvrenamer @@ -0,0 +1 @@ +Subproject commit b1eb1cecebd5b3c5760e71370bed09ca0d0db1d9 diff --git a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php index bf50bde..73e0e54 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php @@ -7,11 +7,16 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug * * @var string */ - const PLUGIN_NAME = "TV"; + const PLUGIN_NAME = "TV"; protected $config; protected $log; + protected $output_dir_cache; + + protected $input_dirs; + protected $output_dir; + public static function create($config, $log) { return new self($config, $log); } @@ -19,18 +24,20 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug protected function __construct($config, $log) { $this->config = $config; $this->log = $log; + + $this->input_dirs = $this->config->get('sources.TV.input'); + $this->output_dir = $this->config->get('sources.TV.output'); } public function run() { DownloadDispatcher_LogEntry::debug($this->log, 'Running TV dispatcher'); // Iterate over source directories, and move matched files to the output directory - $source_dirs = $this->config->get('sources.TV.input-directories'); - foreach ($source_dirs as $dir) { + foreach ($this->input_dirs as $dir) { if (is_dir($dir) && is_readable($dir)) { $iterator = new DownloadDispatcher_Utility_MediaFilesIterator(new DownloadDispatcher_Utility_VisibleFilesIterator(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir)))); foreach ($iterator as /** @var SplFileInfo */ $file) { - $this->process_matched_file($file->getPath(), $file->getFilename()); + $this->processMatchedFile($file->getPath(), $file->getFilename(), $file->getExtension()); } } else { DownloadDispatcher_LogEntry::warning($this->log, "TV input directory '{$dir}' does not exist or cannot be read."); @@ -38,31 +45,177 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug } } - protected function process_matched_file($dir, $file) { + protected function processMatchedFile($dir, $file, $type) { // TODO - Handle movement of the matched file to the correct output directory // Handle direct media files, and also RAR archives DownloadDispatcher_LogEntry::debug($this->log, "Media file: {$file}"); // Check to see if this file has been handled previously - if ($this->check_processed($dir . '/' . $file)) { + if ($this->checkProcessed($dir . '/' . $file)) { DownloadDispatcher_LogEntry::debug($this->log, "Skipping previously seen file"); return; } + $full_output_dir = $this->identifyOutputDir($dir, $file); + if ($full_output_dir) { + if ($this->noDuplicates($full_output_dir, $file)) { + if ($this->copyOutput($type, $dir, $file, $full_output_dir)) { + $this->renameOutput($full_output_dir); + } + } + } + } - protected function identify_output_dir($dir, $file) { + protected function identifyOutputDir($dir, $file) { // TODO - Generate the correct output directory, apply any special case mappings, and ensure the destination exists + if (is_null($this->output_dir_cache)) { + $this->scanOutputDir(); + } + + $normalised_file = $this->normalise($file); + if (array_key_exists($normalised_file, $this->output_dir_cache)) { + $season = $this->season($file); + + $full_output_dir = "{$this->output_dir}/{$this->output_dir_cache[$normalised_file]}/Season {$season}"; + + if (is_dir($full_output_dir)) { + return $full_output_dir; + } + } + + DownloadDispatcher_LogEntry::warning($this->log, "TV output directory for '{$file}' could not be identified; you may need to create one."); + return null; } - protected function identify_duplicate($dir, $file) { - // TODO - Verify that the file we've found hasn't already been processed - // Use the cache to reduce processing overhead - // TODO - Upstream caching + protected function scanOutputDir() { + // Get a list of the series and season directories available in normalised form + DownloadDispatcher_LogEntry::debug($this->log, "Scanning TV output directory ({$this->output_dir})"); + $this->output_dir_cache = array(); + + $series_iterator = new DownloadDispatcher_Utility_VisibleFilesIterator(new DirectoryIterator($this->output_dir)); + foreach ($series_iterator as $series) { + $series_name = $series->getBasename(); + $normalised_series = $this->normalise($series_name); + $this->output_dir_cache[$normalised_series] = $series_name; + } + } - protected function rename_output($dir, $file) { - // TODO - use tvrenamer to update the filenames + protected function normalise($name) { + if (preg_match('/(.*?)([\s\.]us)?([\s\.]+(19|20)\d{2})?[\s\.]+(\d+x\d+|s\d+e\d+|\d{3}).*/i', $name, $matches)) { + $name = $matches[1]; + } + + $name = preg_replace('/[^a-zA-Z0-9]/', ' ', $name); + $name = preg_replace('/ +/', ' ', $name); + $name = strtolower($name); + $name = trim($name); + + return $name; + } + + protected function season($name) { + $set_season = function($a) { + for ($i = 1, $l = count($a); $i < $l; ++$i) { + if ($a[$i]) { + return trim($a[$i], '0'); + } + } + return null; + }; + + if (preg_match('/(\d+)x\d+|s(\d+)e\d+|(?:(?:19|20)\d{2}[\s\.]+)?(\d+)\d{2}/i', $name, $matches)) { + return $set_season($matches); + } else { + return 0; + } + } + + protected function episode($name) { + $set_episode = function($a) { + for ($i = 1, $l = count($a); $i < $l; ++$i) { + if ($a[$i]) { + return trim($a[$i], '0'); + } + } + return null; + }; + + if (preg_match('/\d+x(\d+)|s\d+e(\d+)|(?:(?:19|20)\d{2}[\s\.]+)?\d+(\d{2})/i', $name, $matches)) { + return $set_episode($matches); + } else { + return 0; + } + } + + protected function noDuplicates($dir, $file) { + $episode = $this->episode($file); + + $iterator = new DownloadDispatcher_Utility_MediaFilesIterator(new DownloadDispatcher_Utility_VisibleFilesIterator(new DirectoryIterator($dir))); + foreach ($iterator as /** @var SplFileInfo */ $existing_file) { + $existing_episode = $this->episode($existing_file->getFilename()); + if ($existing_episode == $episode) { + return false; + } + } + + return true; + } + + protected function copyOutput($type, $source_dir, $source_file, $destination_dir) { + switch (strtolower($type)) { + case 'rar': { + DownloadDispatcher_LogEntry::info($this->log, "Unrarring '{$source_file}' into '{$destination_dir}'."); + + $command = "/usr/bin/unrar e -p- -sm8192 -y {$source_dir}/{$source_file}"; + DownloadDispatcher_LogEntry::debug($this->log, "Unrarring '{$source_file}' with command: {$command}"); + + list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $destination_dir); + } break; + + case 'avi': { + // Verify that the file isn't a fake + $safe_source_file = escapeshellarg($source_file); + $command = "file {$safe_source_file}"; + DownloadDispatcher_LogEntry::debug($this->log, "Verifying '{$source_file}' contents with command: {$command}"); + list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $source_dir); + var_dump($code, $output, $error); + + if (preg_match('/Microsoft ASF/', $output)) { + DownloadDispatcher_LogEntry::warning($this->log, "Skipping '{$source_dir}/{$source_file}' due to dubious contents."); + return false; + } + + } // continue into the next case + default: { + DownloadDispatcher_LogEntry::info($this->log, "Copying '{$source_file}' to '{$destination_dir}'."); + copy("{$source_dir}/${source_file}", "{$destination_dir}/{$source_file}"); + } + } + + return true; + } + + protected function renameOutput($dir) { + $cwd = getcwd(); + + $command = <<log, "Executing tvrenamer command in '{$dir}': {$command}"); + list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $dir); + var_dump($code, $output, $error); } } diff --git a/source/lib/DownloadDispatcher/Source/PluginBase.class.php b/source/lib/DownloadDispatcher/Source/PluginBase.class.php index 1ec4ed9..6252eb8 100644 --- a/source/lib/DownloadDispatcher/Source/PluginBase.class.php +++ b/source/lib/DownloadDispatcher/Source/PluginBase.class.php @@ -5,17 +5,17 @@ class DownloadDispatcher_Source_PluginBase extends DownloadDispatcher_PluginBase static protected $cache; static protected $source_cache = null; - static protected $cache_file = 'source_cache'; + static protected $source_cache_file = 'source_cache'; static protected $cache_lifetime = 86400; - protected function init_cache() { + protected function initSourceCache() { if ( ! static::$cache) { static::$cache = DownloadDispatcher_Main::instance()->cache(); } if (is_null(static::$source_cache)) { try { - static::$source_cache = static::$cache->fetch(static::$cache_file, static::$cache_lifetime); + static::$source_cache = static::$cache->fetch(static::$source_cache_file, static::$cache_lifetime); } catch (SihnonFramework_Exception_CacheObjectNotFound $e) { static::$source_cache = array(); } @@ -26,18 +26,18 @@ class DownloadDispatcher_Source_PluginBase extends DownloadDispatcher_PluginBase } } - protected function mark_processed($file) { - $this->init_cache(); + protected function markProcessed($file) { + $this->initSourceCache(); if ( ! in_array($file, static::$source_cache[get_called_class()])) { static::$source_cache[get_called_class()][] = $file; } - static::$cache->store($cache_file, static::$source_cache); + static::$cache->store(static::$source_cache_file, static::$source_cache); } - protected function check_processed($file) { - $this->init_cache(); + protected function checkProcessed($file) { + $this->initSourceCache(); return in_array($file, static::$source_cache[get_called_class()]); } From 0eabec90babb2349c4c1f0ad8b832c1ecce7b25f Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Sat, 31 Dec 2011 01:30:06 +0000 Subject: [PATCH 05/14] Remove unnecessary variables and debug output --- source/lib/DownloadDispatcher/Source/Plugin/TV.class.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php index 73e0e54..b38b126 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php @@ -171,7 +171,7 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug $command = "/usr/bin/unrar e -p- -sm8192 -y {$source_dir}/{$source_file}"; DownloadDispatcher_LogEntry::debug($this->log, "Unrarring '{$source_file}' with command: {$command}"); - list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $destination_dir); + DownloadDispatcher_ForegroundTask::execute($command, $destination_dir); } break; case 'avi': { @@ -180,7 +180,6 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug $command = "file {$safe_source_file}"; DownloadDispatcher_LogEntry::debug($this->log, "Verifying '{$source_file}' contents with command: {$command}"); list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $source_dir); - var_dump($code, $output, $error); if (preg_match('/Microsoft ASF/', $output)) { DownloadDispatcher_LogEntry::warning($this->log, "Skipping '{$source_dir}/{$source_file}' due to dubious contents."); @@ -214,8 +213,7 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug EOSH; DownloadDispatcher_LogEntry::debug($this->log, "Executing tvrenamer command in '{$dir}': {$command}"); - list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $dir); - var_dump($code, $output, $error); + DownloadDispatcher_ForegroundTask::execute($command, $dir); } } From c13090b47294b96424c46302ce993ec80b37706c Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Sat, 31 Dec 2011 02:10:06 +0000 Subject: [PATCH 06/14] Use a Daemon object for process locking --- source/lib/DownloadDispatcher/Main.class.php | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 source/lib/DownloadDispatcher/Main.class.php diff --git a/source/lib/DownloadDispatcher/Main.class.php b/source/lib/DownloadDispatcher/Main.class.php new file mode 100644 index 0000000..8334745 --- /dev/null +++ b/source/lib/DownloadDispatcher/Main.class.php @@ -0,0 +1,25 @@ +daemon = new DownloadDispatcher_Daemon($this->config); + + } catch (SihnonFramework_Exception_AlreadyRunning $e) { + DownloadDispatcher_LogEntry::error($this->log, "Another instance is already running, exiting this process now."); + exit(0); + } + } + +} + +?> \ No newline at end of file From 48bfedf6dd0166b3e5cc95b14de0ea01b013f0ea Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Sat, 31 Dec 2011 02:25:18 +0000 Subject: [PATCH 07/14] Add addiitonal log output and cache completed entries --- source/lib/DownloadDispatcher/Source/Plugin/TV.class.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php index b38b126..f07d637 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php @@ -61,8 +61,15 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug if ($this->noDuplicates($full_output_dir, $file)) { if ($this->copyOutput($type, $dir, $file, $full_output_dir)) { $this->renameOutput($full_output_dir); + } else { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$file}' to the destination directory."); } + } else { + DownloadDispatcher_LogEntry::info($this->log, "Skipping duplicate file '{$file}'."); } + + // This file has been dealt with, so no need to look at it in subsequent operations + $this->markProcessed($file); } } From babeb575eb88d41028a2cb1e892d39f20cbbd570 Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Sat, 31 Dec 2011 02:29:28 +0000 Subject: [PATCH 08/14] Serialise output before caching --- source/lib/DownloadDispatcher/Source/PluginBase.class.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/lib/DownloadDispatcher/Source/PluginBase.class.php b/source/lib/DownloadDispatcher/Source/PluginBase.class.php index 6252eb8..47360ca 100644 --- a/source/lib/DownloadDispatcher/Source/PluginBase.class.php +++ b/source/lib/DownloadDispatcher/Source/PluginBase.class.php @@ -15,7 +15,7 @@ class DownloadDispatcher_Source_PluginBase extends DownloadDispatcher_PluginBase if (is_null(static::$source_cache)) { try { - static::$source_cache = static::$cache->fetch(static::$source_cache_file, static::$cache_lifetime); + static::$source_cache = unserialize(static::$cache->fetch(static::$source_cache_file, static::$cache_lifetime)); } catch (SihnonFramework_Exception_CacheObjectNotFound $e) { static::$source_cache = array(); } @@ -33,7 +33,7 @@ class DownloadDispatcher_Source_PluginBase extends DownloadDispatcher_PluginBase static::$source_cache[get_called_class()][] = $file; } - static::$cache->store(static::$source_cache_file, static::$source_cache); + static::$cache->store(static::$source_cache_file, serialize(static::$source_cache)); } protected function checkProcessed($file) { From fe0a8ceedd39ea1abce2001df2af9b48e7cb818e Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Sat, 7 Jan 2012 19:47:58 +0000 Subject: [PATCH 09/14] Escape filename when unrarring and add better failure log output --- .../Source/Plugin/TV.class.php | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php index f07d637..109cdb1 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php @@ -175,10 +175,15 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug case 'rar': { DownloadDispatcher_LogEntry::info($this->log, "Unrarring '{$source_file}' into '{$destination_dir}'."); - $command = "/usr/bin/unrar e -p- -sm8192 -y {$source_dir}/{$source_file}"; + $safe_source_file = escapeshellarg("{$source_dir}/{$source_file}"); + $command = "/usr/bin/unrar e -p- -sm8192 -y {$safe_source_file}"; DownloadDispatcher_LogEntry::debug($this->log, "Unrarring '{$source_file}' with command: {$command}"); - DownloadDispatcher_ForegroundTask::execute($command, $destination_dir); + list ($code,$output,$error) = DownloadDispatcher_ForegroundTask::execute($command, $destination_dir); + if ($code != 0) { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to unrar '{$source_dir}/{$source_file}'."); + return false; + } } break; case 'avi': { @@ -187,6 +192,10 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug $command = "file {$safe_source_file}"; DownloadDispatcher_LogEntry::debug($this->log, "Verifying '{$source_file}' contents with command: {$command}"); list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $source_dir); + if ($code != 0) { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to determine contents of '{$source_dir}/{$source_file}'."); + return false; + } if (preg_match('/Microsoft ASF/', $output)) { DownloadDispatcher_LogEntry::warning($this->log, "Skipping '{$source_dir}/{$source_file}' due to dubious contents."); @@ -196,7 +205,11 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug } // continue into the next case default: { DownloadDispatcher_LogEntry::info($this->log, "Copying '{$source_file}' to '{$destination_dir}'."); - copy("{$source_dir}/${source_file}", "{$destination_dir}/{$source_file}"); + $result = copy("{$source_dir}/${source_file}", "{$destination_dir}/{$source_file}"); + if ( ! $result) { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$source_dir}/{$source_file}' to output directory '{$destination_dir}'."); + return false; + } } } From a26967a2726472ef03d7c5a5eb88604633f8aa7e Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Sat, 7 Jan 2012 20:01:25 +0000 Subject: [PATCH 10/14] Permit dubious episode matches for seasons >10 --- source/lib/DownloadDispatcher/Source/Plugin/TV.class.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php index 109cdb1..d77612d 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php @@ -110,7 +110,7 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug } protected function normalise($name) { - if (preg_match('/(.*?)([\s\.]us)?([\s\.]+(19|20)\d{2})?[\s\.]+(\d+x\d+|s\d+e\d+|\d{3}).*/i', $name, $matches)) { + if (preg_match('/(.*?)([\s\.]us)?([\s\.]+(19|20)\d{2})?[\s\.]+(\d+x\d+|s\d+e\d+|\d{3,4}).*/i', $name, $matches)) { $name = $matches[1]; } From 08ec4e7d45ef0d605be617a5042cd9cd2a323097 Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Mon, 9 Jan 2012 20:14:14 +0000 Subject: [PATCH 11/14] Fix incorrect removal of trailing zeroes from season/episode numbers --- source/lib/DownloadDispatcher/Source/Plugin/TV.class.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php index d77612d..e8ed044 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php @@ -91,7 +91,7 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug } } - DownloadDispatcher_LogEntry::warning($this->log, "TV output directory for '{$file}' could not be identified; you may need to create one."); + DownloadDispatcher_LogEntry::warning($this->log, "TV output directory for '{$file}' ({$normalised_file}) could not be identified; you may need to create one."); return null; } @@ -106,7 +106,6 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug $normalised_series = $this->normalise($series_name); $this->output_dir_cache[$normalised_series] = $series_name; } - } protected function normalise($name) { @@ -126,7 +125,7 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug $set_season = function($a) { for ($i = 1, $l = count($a); $i < $l; ++$i) { if ($a[$i]) { - return trim($a[$i], '0'); + return ltrim($a[$i], '0'); } } return null; @@ -143,7 +142,7 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug $set_episode = function($a) { for ($i = 1, $l = count($a); $i < $l; ++$i) { if ($a[$i]) { - return trim($a[$i], '0'); + return ltrim($a[$i], '0'); } } return null; @@ -238,4 +237,4 @@ EOSH; } -?> \ No newline at end of file +?> From 37bb0dde39eb1519db7fbac5c22307945e15d862 Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Tue, 10 Jan 2012 01:31:42 +0000 Subject: [PATCH 12/14] Add exception classes for Source Plugins --- source/lib/DownloadDispatcher/Exceptions.class.php | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 source/lib/DownloadDispatcher/Exceptions.class.php diff --git a/source/lib/DownloadDispatcher/Exceptions.class.php b/source/lib/DownloadDispatcher/Exceptions.class.php new file mode 100644 index 0000000..5dd1223 --- /dev/null +++ b/source/lib/DownloadDispatcher/Exceptions.class.php @@ -0,0 +1,10 @@ + \ No newline at end of file From e02869e66b00b68e52f196b980dcd2b9dde9824f Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Tue, 10 Jan 2012 01:32:01 +0000 Subject: [PATCH 13/14] Add optional setting to disable an rsync instance --- source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php b/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php index e5e61b4..f5108d3 100644 --- a/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php +++ b/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php @@ -13,6 +13,7 @@ class DownloadDispatcher_Sync_Plugin_Rsync extends DownloadDispatcher_PluginBase protected $log; protected $instance; + protected $enabled; protected $options; protected $source; protected $destination; @@ -26,12 +27,17 @@ class DownloadDispatcher_Sync_Plugin_Rsync extends DownloadDispatcher_PluginBase $this->log = $log; $this->instance = $instance; + $this->enabled = $this->config->get("sync.Rsync.{$this->instance}.enabled", true); $this->options = $this->config->get("sync.Rsync.{$this->instance}.options"); $this->source = $this->config->get("sync.Rsync.{$this->instance}.source"); $this->destination = $this->config->get("sync.Rsync.{$this->instance}.destination"); } public function run() { + if ( ! $this->enabled) { + return; + } + DownloadDispatcher_LogEntry::debug($this->log, "Running Rsync synchroniser: '{$this->instance}'"); $command = "/usr/bin/rsync {$this->options} '{$this->source}' '{$this->destination}'"; From 75342fe701b854a7ece5808fec42226952302398 Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Tue, 10 Jan 2012 01:32:56 +0000 Subject: [PATCH 14/14] Add flexget --series-forget support, exceptionise error handling Code now depends upon pecl-http. --- .../Source/Plugin/TV.class.php | 105 ++++++++++++------ 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php index d77612d..cbdd0c0 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php @@ -48,30 +48,46 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug protected function processMatchedFile($dir, $file, $type) { // TODO - Handle movement of the matched file to the correct output directory // Handle direct media files, and also RAR archives - DownloadDispatcher_LogEntry::debug($this->log, "Media file: {$file}"); - - // Check to see if this file has been handled previously - if ($this->checkProcessed($dir . '/' . $file)) { - DownloadDispatcher_LogEntry::debug($this->log, "Skipping previously seen file"); - return; - } + DownloadDispatcher_LogEntry::debug($this->log, "Media file: '{$file}'."); - $full_output_dir = $this->identifyOutputDir($dir, $file); - if ($full_output_dir) { - if ($this->noDuplicates($full_output_dir, $file)) { - if ($this->copyOutput($type, $dir, $file, $full_output_dir)) { - $this->renameOutput($full_output_dir); - } else { - DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$file}' to the destination directory."); - } - } else { - DownloadDispatcher_LogEntry::info($this->log, "Skipping duplicate file '{$file}'."); + try { + + // Check to see if this file has been handled previously + if ($this->checkProcessed($dir . '/' . $file)) { + throw new DownloadDispatcher_Exception_PreviouslySeenContent($file); } + $full_output_dir = $this->identifyOutputDir($dir, $file); + + $this->checkDuplicates($full_output_dir, $file); + + $this->copyOutput($type, $dir, $file, $full_output_dir); + + $this->renameOutput($full_output_dir); + // This file has been dealt with, so no need to look at it in subsequent operations $this->markProcessed($file); + + } catch (DownloadDispatcher_Exception_PreviouslySeenContent $e) { + DownloadDispatcher_LogEntry::debug($this->log, "Skipping previously seen file '{$e->getMessage()}'."); + + } catch (DownloadDispatcher_Exception_UnidentifiedContent $e) { + DownloadDispatcher_LogEntry::warning($this->log, "TV output directory for '{$e->getMessage()}' could not be identified; you may need to create one."); + + } catch (DownloadDispatcher_Exception_UnacceptableContent $e) { + DownloadDispatcher_LogEntry::warning($this->log, "Skipping '{$e->getMessage()}' due to dubious contents."); + + // Forget the download upstream so a new copy can be fetched + $file = $e->getMessage(); + $this->forgetDownload($this->normalise($file), $this->season($file), $this->episode($file)); + + } catch (DownloadDispatcher_Exception_DuplicateContent $e) { + DownloadDispatcher_LogEntry::info($this->log, "Skipping duplicate file '{$e->getMessage()}'."); + + } catch (DownloadDispatcher_Exception_UnprocesseableContent $e) { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$e->getMessage()}' to the destination directory."); + } - } protected function identifyOutputDir($dir, $file) { @@ -91,8 +107,7 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug } } - DownloadDispatcher_LogEntry::warning($this->log, "TV output directory for '{$file}' could not be identified; you may need to create one."); - return null; + throw new DownloadDispatcher_Exception_UnidentifiedContent($file); } protected function scanOutputDir() { @@ -156,18 +171,16 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug } } - protected function noDuplicates($dir, $file) { + protected function checkDuplicates($dir, $file) { $episode = $this->episode($file); $iterator = new DownloadDispatcher_Utility_MediaFilesIterator(new DownloadDispatcher_Utility_VisibleFilesIterator(new DirectoryIterator($dir))); foreach ($iterator as /** @var SplFileInfo */ $existing_file) { $existing_episode = $this->episode($existing_file->getFilename()); if ($existing_episode == $episode) { - return false; + throw new DownloadDispatcher_Exception_DuplicateContent($file); } } - - return true; } protected function copyOutput($type, $source_dir, $source_file, $destination_dir) { @@ -180,9 +193,11 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug DownloadDispatcher_LogEntry::debug($this->log, "Unrarring '{$source_file}' with command: {$command}"); list ($code,$output,$error) = DownloadDispatcher_ForegroundTask::execute($command, $destination_dir); - if ($code != 0) { + if ($code == 3) { + throw new DownloadDispatcher_Exception_UnacceptableContent($source_file); + } else if ($code != 0) { DownloadDispatcher_LogEntry::warning($this->log, "Failed to unrar '{$source_dir}/{$source_file}'."); - return false; + throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file); } } break; @@ -194,21 +209,20 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $source_dir); if ($code != 0) { DownloadDispatcher_LogEntry::warning($this->log, "Failed to determine contents of '{$source_dir}/{$source_file}'."); - return false; + throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file); } if (preg_match('/Microsoft ASF/', $output)) { - DownloadDispatcher_LogEntry::warning($this->log, "Skipping '{$source_dir}/{$source_file}' due to dubious contents."); - return false; + throw new DownloadDispatcher_Exception_UnacceptableContent($source_file); } } // continue into the next case default: { DownloadDispatcher_LogEntry::info($this->log, "Copying '{$source_file}' to '{$destination_dir}'."); - $result = copy("{$source_dir}/${source_file}", "{$destination_dir}/{$source_file}"); + $result = copy("{$source_dir}/{$source_file}", "{$destination_dir}/{$source_file}"); if ( ! $result) { DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$source_dir}/{$source_file}' to output directory '{$destination_dir}'."); - return false; + throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file); } } } @@ -236,6 +250,35 @@ EOSH; DownloadDispatcher_ForegroundTask::execute($command, $dir); } + protected function forgetDownload($series, $season, $episode) { + $base_url = $this->config->get('sources.TV.flexget-url'); + $username = $this->config->get('sources.TV.flexget-username'); + $password = $this->config->get('sources.TV.flexget-password'); + + $url = "{$base_url}execute/"; + $data = array( + 'options' => "--series-forget '{$series}' 's{$season}e{$episode}'", + 'submit' => 'Start Execution', + ); + + DownloadDispatcher_LogEntry::debug($this->log, "Sending flexget series-forget command to {$url} with options '{$data['options']}'."); + + $request = new HttpRequest($url, HTTP_METH_POST, array( + 'httpauth' => "{$username}:{$password}", + 'httpauthtype' => HTTP_AUTH_BASIC, + )); + $request->setPostFields($data); + + $response = $request->send(); + DownloadDispatcher_LogEntry::debug($this->log, "Response code: {$response->getResponseCode()}."); + + if ($response->getResponseCode() == 200) { + DownloadDispatcher_LogEntry::info($this->log, "Successfully made flexget forget about {$series} s{$season}e{$episode}."); + } else { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to make flexget forget about {$series} s{$season}e{$episode}."); + } + } + } ?> \ No newline at end of file