diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0c825a5 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "source/3rdparty/tvrenamer"] + path = source/3rdparty/tvrenamer + url = https://github.com/meermanr/TVSeriesRenamer.git diff --git a/source/3rdparty/tvrenamer b/source/3rdparty/tvrenamer new file mode 160000 index 0000000..b1eb1ce --- /dev/null +++ b/source/3rdparty/tvrenamer @@ -0,0 +1 @@ +Subproject commit b1eb1cecebd5b3c5760e71370bed09ca0d0db1d9 diff --git a/source/lib/DownloadDispatcher/Exceptions.class.php b/source/lib/DownloadDispatcher/Exceptions.class.php new file mode 100644 index 0000000..5dd1223 --- /dev/null +++ b/source/lib/DownloadDispatcher/Exceptions.class.php @@ -0,0 +1,10 @@ + \ No newline at end of file diff --git a/source/lib/DownloadDispatcher/Main.class.php b/source/lib/DownloadDispatcher/Main.class.php new file mode 100644 index 0000000..8334745 --- /dev/null +++ b/source/lib/DownloadDispatcher/Main.class.php @@ -0,0 +1,25 @@ +daemon = new DownloadDispatcher_Daemon($this->config); + + } catch (SihnonFramework_Exception_AlreadyRunning $e) { + DownloadDispatcher_LogEntry::error($this->log, "Another instance is already running, exiting this process now."); + exit(0); + } + } + +} + +?> \ No newline at end of file diff --git a/source/lib/DownloadDispatcher/Processor.class.php b/source/lib/DownloadDispatcher/Processor.class.php index e7a766d..a5b7ac5 100644 --- a/source/lib/DownloadDispatcher/Processor.class.php +++ b/source/lib/DownloadDispatcher/Processor.class.php @@ -23,7 +23,7 @@ class DownloadDispatcher_Processor { $plugin = DownloadDispatcher_Sync_PluginFactory::create($plugin_name, $config, $log, $instance); $plugin->run(); - } catch(SihnonFramework_Exception_LogException $e) { + } catch(SihnonFramework_Exception_PluginException $e) { SihnonFramework_LogEntry::warning($log, $e->getMessage()); } } @@ -31,13 +31,14 @@ class DownloadDispatcher_Processor { // Find the list of available source plugins DownloadDispatcher_Source_PluginFactory::scan(); - $source_plugins = DownloadDispatcher_Source_PluginFactory::getValidPlugins(); - - $enabled_plugins = $config->get('sources'); + $source_plugins = $config->get('sources'); foreach ($source_plugins as $plugin_name) { - if (in_array($plugin_name, $enabled_plugins)) { + try { $plugin = DownloadDispatcher_Source_PluginFactory::create($plugin_name, $config, $log); $plugin->run(); + + } catch(DownloadDispatcher_Exception_PluginException $e) { + SihnonFramework_LogEntry::warning($log, $e->getMessage()); } } } diff --git a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php index bf50bde..554ebb3 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/TV.class.php @@ -7,11 +7,16 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug * * @var string */ - const PLUGIN_NAME = "TV"; + const PLUGIN_NAME = "TV"; protected $config; protected $log; + protected $output_dir_cache; + + protected $input_dirs; + protected $output_dir; + public static function create($config, $log) { return new self($config, $log); } @@ -19,18 +24,20 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug protected function __construct($config, $log) { $this->config = $config; $this->log = $log; + + $this->input_dirs = $this->config->get('sources.TV.input'); + $this->output_dir = $this->config->get('sources.TV.output'); } public function run() { DownloadDispatcher_LogEntry::debug($this->log, 'Running TV dispatcher'); // Iterate over source directories, and move matched files to the output directory - $source_dirs = $this->config->get('sources.TV.input-directories'); - foreach ($source_dirs as $dir) { + foreach ($this->input_dirs as $dir) { if (is_dir($dir) && is_readable($dir)) { $iterator = new DownloadDispatcher_Utility_MediaFilesIterator(new DownloadDispatcher_Utility_VisibleFilesIterator(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir)))); foreach ($iterator as /** @var SplFileInfo */ $file) { - $this->process_matched_file($file->getPath(), $file->getFilename()); + $this->processMatchedFile($file->getPath(), $file->getFilename(), $file->getExtension()); } } else { DownloadDispatcher_LogEntry::warning($this->log, "TV input directory '{$dir}' does not exist or cannot be read."); @@ -38,33 +45,239 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug } } - protected function process_matched_file($dir, $file) { + protected function processMatchedFile($dir, $file, $type) { // TODO - Handle movement of the matched file to the correct output directory // Handle direct media files, and also RAR archives - DownloadDispatcher_LogEntry::debug($this->log, "Media file: {$file}"); + DownloadDispatcher_LogEntry::debug($this->log, "Media file: '{$file}'."); + + try { - // Check to see if this file has been handled previously - if ($this->check_processed($dir . '/' . $file)) { - DownloadDispatcher_LogEntry::debug($this->log, "Skipping previously seen file"); - return; + // Check to see if this file has been handled previously + if ($this->checkProcessed($dir . '/' . $file)) { + throw new DownloadDispatcher_Exception_PreviouslySeenContent($file); + } + + $full_output_dir = $this->identifyOutputDir($dir, $file); + + $this->checkDuplicates($full_output_dir, $file); + + $this->copyOutput($type, $dir, $file, $full_output_dir); + + $this->renameOutput($full_output_dir); + + // This file has been dealt with, so no need to look at it in subsequent operations + $this->markProcessed($file); + + } catch (DownloadDispatcher_Exception_PreviouslySeenContent $e) { + DownloadDispatcher_LogEntry::debug($this->log, "Skipping previously seen file '{$e->getMessage()}'."); + + } catch (DownloadDispatcher_Exception_UnidentifiedContent $e) { + DownloadDispatcher_LogEntry::warning($this->log, "TV output directory for '{$e->getMessage()}' could not be identified; you may need to create one."); + + } catch (DownloadDispatcher_Exception_UnacceptableContent $e) { + DownloadDispatcher_LogEntry::warning($this->log, "Skipping '{$e->getMessage()}' due to dubious contents."); + + // Forget the download upstream so a new copy can be fetched + $file = $e->getMessage(); + $this->forgetDownload($this->normalise($file), $this->season($file), $this->episode($file)); + + } catch (DownloadDispatcher_Exception_DuplicateContent $e) { + DownloadDispatcher_LogEntry::info($this->log, "Skipping duplicate file '{$e->getMessage()}'."); + + } catch (DownloadDispatcher_Exception_UnprocesseableContent $e) { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$e->getMessage()}' to the destination directory."); + + } + } + + protected function identifyOutputDir($dir, $file) { + // TODO - Generate the correct output directory, apply any special case mappings, and ensure the destination exists + if (is_null($this->output_dir_cache)) { + $this->scanOutputDir(); } + $normalised_file = $this->normalise($file); + if (array_key_exists($normalised_file, $this->output_dir_cache)) { + $season = $this->season($file); + + $full_output_dir = "{$this->output_dir}/{$this->output_dir_cache[$normalised_file]}/Season {$season}"; + + if (is_dir($full_output_dir)) { + return $full_output_dir; + } + } + + throw new DownloadDispatcher_Exception_UnidentifiedContent($file); } - protected function identify_output_dir($dir, $file) { - // TODO - Generate the correct output directory, apply any special case mappings, and ensure the destination exists + protected function scanOutputDir() { + // Get a list of the series and season directories available in normalised form + DownloadDispatcher_LogEntry::debug($this->log, "Scanning TV output directory ({$this->output_dir})"); + $this->output_dir_cache = array(); + + $series_iterator = new DownloadDispatcher_Utility_VisibleFilesIterator(new DirectoryIterator($this->output_dir)); + foreach ($series_iterator as $series) { + $series_name = $series->getBasename(); + $normalised_series = $this->normalise($series_name); + $this->output_dir_cache[$normalised_series] = $series_name; + } } - protected function identify_duplicate($dir, $file) { - // TODO - Verify that the file we've found hasn't already been processed - // Use the cache to reduce processing overhead - // TODO - Upstream caching + protected function normalise($name) { + if (preg_match('/(.*?)([\s\.]us)?([\s\.]+(19|20)\d{2})?[\s\.]+(\d+x\d+|s\d+e\d+|\d{3,4}).*/i', $name, $matches)) { + $name = $matches[1]; + } + + $name = preg_replace('/[^a-zA-Z0-9]/', ' ', $name); + $name = preg_replace('/ +/', ' ', $name); + $name = strtolower($name); + $name = trim($name); + + return $name; } - protected function rename_output($dir, $file) { - // TODO - use tvrenamer to update the filenames + protected function season($name) { + $set_season = function($a) { + for ($i = 1, $l = count($a); $i < $l; ++$i) { + if ($a[$i]) { + return ltrim($a[$i], '0'); + } + } + return null; + }; + + if (preg_match('/(\d+)x\d+|s(\d+)e\d+|(?:(?:19|20)\d{2}[\s\.]+)?(\d+)\d{2}/i', $name, $matches)) { + return $set_season($matches); + } else { + return 0; + } + } + + protected function episode($name) { + $set_episode = function($a) { + for ($i = 1, $l = count($a); $i < $l; ++$i) { + if ($a[$i]) { + return ltrim($a[$i], '0'); + } + } + return null; + }; + + if (preg_match('/\d+x(\d+)|s\d+e(\d+)|(?:(?:19|20)\d{2}[\s\.]+)?\d+(\d{2})/i', $name, $matches)) { + return $set_episode($matches); + } else { + return 0; + } + } + + protected function checkDuplicates($dir, $file) { + $episode = $this->episode($file); + + $iterator = new DownloadDispatcher_Utility_MediaFilesIterator(new DownloadDispatcher_Utility_VisibleFilesIterator(new DirectoryIterator($dir))); + foreach ($iterator as /** @var SplFileInfo */ $existing_file) { + $existing_episode = $this->episode($existing_file->getFilename()); + if ($existing_episode == $episode) { + throw new DownloadDispatcher_Exception_DuplicateContent($file); + } + } + } + + protected function copyOutput($type, $source_dir, $source_file, $destination_dir) { + switch (strtolower($type)) { + case 'rar': { + DownloadDispatcher_LogEntry::info($this->log, "Unrarring '{$source_file}' into '{$destination_dir}'."); + + $safe_source_file = escapeshellarg("{$source_dir}/{$source_file}"); + $command = "/usr/bin/unrar e -p- -sm8192 -y {$safe_source_file}"; + DownloadDispatcher_LogEntry::debug($this->log, "Unrarring '{$source_file}' with command: {$command}"); + + list ($code,$output,$error) = DownloadDispatcher_ForegroundTask::execute($command, $destination_dir); + if ($code == 3) { + throw new DownloadDispatcher_Exception_UnacceptableContent($source_file); + } else if ($code != 0) { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to unrar '{$source_dir}/{$source_file}'."); + throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file); + } + } break; + + case 'avi': { + // Verify that the file isn't a fake + $safe_source_file = escapeshellarg($source_file); + $command = "file {$safe_source_file}"; + DownloadDispatcher_LogEntry::debug($this->log, "Verifying '{$source_file}' contents with command: {$command}"); + list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $source_dir); + if ($code != 0) { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to determine contents of '{$source_dir}/{$source_file}'."); + throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file); + } + + if (preg_match('/Microsoft ASF/', $output)) { + throw new DownloadDispatcher_Exception_UnacceptableContent($source_file); + } + + } // continue into the next case + default: { + DownloadDispatcher_LogEntry::info($this->log, "Copying '{$source_file}' to '{$destination_dir}'."); + $result = copy("{$source_dir}/{$source_file}", "{$destination_dir}/{$source_file}"); + if ( ! $result) { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$source_dir}/{$source_file}' to output directory '{$destination_dir}'."); + throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file); + } + } + } + + return true; + } + + protected function renameOutput($dir) { + $cwd = getcwd(); + + $command = <<log, "Executing tvrenamer command in '{$dir}': {$command}"); + DownloadDispatcher_ForegroundTask::execute($command, $dir); + } + + protected function forgetDownload($series, $season, $episode) { + $base_url = $this->config->get('sources.TV.flexget-url'); + $username = $this->config->get('sources.TV.flexget-username'); + $password = $this->config->get('sources.TV.flexget-password'); + + $url = "{$base_url}execute/"; + $data = array( + 'options' => "--series-forget '{$series}' 's{$season}e{$episode}'", + 'submit' => 'Start Execution', + ); + + DownloadDispatcher_LogEntry::debug($this->log, "Sending flexget series-forget command to {$url} with options '{$data['options']}'."); + + $request = new HttpRequest($url, HTTP_METH_POST, array( + 'httpauth' => "{$username}:{$password}", + 'httpauthtype' => HTTP_AUTH_BASIC, + )); + $request->setPostFields($data); + + $response = $request->send(); + DownloadDispatcher_LogEntry::debug($this->log, "Response code: {$response->getResponseCode()}."); + + if ($response->getResponseCode() == 200) { + DownloadDispatcher_LogEntry::info($this->log, "Successfully made flexget forget about {$series} s{$season}e{$episode}."); + } else { + DownloadDispatcher_LogEntry::warning($this->log, "Failed to make flexget forget about {$series} s{$season}e{$episode}."); + } } } -?> \ No newline at end of file +?> diff --git a/source/lib/DownloadDispatcher/Source/PluginBase.class.php b/source/lib/DownloadDispatcher/Source/PluginBase.class.php index 72567c3..47360ca 100644 --- a/source/lib/DownloadDispatcher/Source/PluginBase.class.php +++ b/source/lib/DownloadDispatcher/Source/PluginBase.class.php @@ -2,27 +2,42 @@ class DownloadDispatcher_Source_PluginBase extends DownloadDispatcher_PluginBase { - static protected $source_cache = array(); + static protected $cache; - protected function init_cache() { + static protected $source_cache = null; + static protected $source_cache_file = 'source_cache'; + static protected $cache_lifetime = 86400; + + protected function initSourceCache() { + if ( ! static::$cache) { + static::$cache = DownloadDispatcher_Main::instance()->cache(); + } + + if (is_null(static::$source_cache)) { + try { + static::$source_cache = unserialize(static::$cache->fetch(static::$source_cache_file, static::$cache_lifetime)); + } catch (SihnonFramework_Exception_CacheObjectNotFound $e) { + static::$source_cache = array(); + } + } + if ( ! array_key_exists(get_called_class(), static::$source_cache)) { - // TODO - attempt to load data from persistent storage static::$source_cache[get_called_class()] = array(); } } - protected function mark_processed($file) { - $this->init_cache(); + protected function markProcessed($file) { + $this->initSourceCache(); if ( ! in_array($file, static::$source_cache[get_called_class()])) { static::$source_cache[get_called_class()][] = $file; } - // TODO - flush cache to persistent storage + static::$cache->store(static::$source_cache_file, serialize(static::$source_cache)); } - protected function check_processed($file) { - $this->init_cache(); + protected function checkProcessed($file) { + $this->initSourceCache(); return in_array($file, static::$source_cache[get_called_class()]); } diff --git a/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php b/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php index e5e61b4..f5108d3 100644 --- a/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php +++ b/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php @@ -13,6 +13,7 @@ class DownloadDispatcher_Sync_Plugin_Rsync extends DownloadDispatcher_PluginBase protected $log; protected $instance; + protected $enabled; protected $options; protected $source; protected $destination; @@ -26,12 +27,17 @@ class DownloadDispatcher_Sync_Plugin_Rsync extends DownloadDispatcher_PluginBase $this->log = $log; $this->instance = $instance; + $this->enabled = $this->config->get("sync.Rsync.{$this->instance}.enabled", true); $this->options = $this->config->get("sync.Rsync.{$this->instance}.options"); $this->source = $this->config->get("sync.Rsync.{$this->instance}.source"); $this->destination = $this->config->get("sync.Rsync.{$this->instance}.destination"); } public function run() { + if ( ! $this->enabled) { + return; + } + DownloadDispatcher_LogEntry::debug($this->log, "Running Rsync synchroniser: '{$this->instance}'"); $command = "/usr/bin/rsync {$this->options} '{$this->source}' '{$this->destination}'"; diff --git a/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php b/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php index 1ce7df2..b5c8d62 100644 --- a/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php +++ b/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php @@ -1,8 +1,14 @@ current()->getFilename()); + public function accept() { + $filename = $this->current()->getFilename(); + if (preg_match('/^sample/', $filename)) { + return false; + } + if (preg_match('/(?