From 579f735b493f310fe5c66e59221fca92a3bfbdfc Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Mon, 12 Dec 2011 23:57:05 +0000 Subject: [PATCH 1/3] Add Sync plugins Add support for synchronising remote content before running the source dispatchers. First plugin adds Rsync support. --- .../DownloadDispatcher/Processor.class.php | 22 +++++++-- .../DownloadDispatcher/Sync/IPlugin.class.php | 16 +++++++ .../Sync/Plugin/Rsync.class.php | 48 +++++++++++++++++++ .../Sync/PluginFactory.class.php | 30 ++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 source/lib/DownloadDispatcher/Sync/IPlugin.class.php create mode 100644 source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php create mode 100644 source/lib/DownloadDispatcher/Sync/PluginFactory.class.php diff --git a/source/lib/DownloadDispatcher/Processor.class.php b/source/lib/DownloadDispatcher/Processor.class.php index c52313e..e7a766d 100644 --- a/source/lib/DownloadDispatcher/Processor.class.php +++ b/source/lib/DownloadDispatcher/Processor.class.php @@ -13,12 +13,28 @@ class DownloadDispatcher_Processor { $config = $main->config(); $log = $main->log(); - // Find the list of available plugins + // Find the list of available Sync plugins + $sync_plugins = $config->get('sync'); + foreach ($sync_plugins as $plugin_name) { + // Get a list of all the instances of this plugin to be used + $instances = $config->get("sync.{$plugin_name}"); + foreach ($instances as $instance) { + try { + $plugin = DownloadDispatcher_Sync_PluginFactory::create($plugin_name, $config, $log, $instance); + $plugin->run(); + + } catch(SihnonFramework_Exception_LogException $e) { + SihnonFramework_LogEntry::warning($log, $e->getMessage()); + } + } + } + + // Find the list of available source plugins DownloadDispatcher_Source_PluginFactory::scan(); - $plugins = DownloadDispatcher_Source_PluginFactory::getValidPlugins(); + $source_plugins = DownloadDispatcher_Source_PluginFactory::getValidPlugins(); $enabled_plugins = $config->get('sources'); - foreach ($plugins as $plugin_name) { + foreach ($source_plugins as $plugin_name) { if (in_array($plugin_name, $enabled_plugins)) { $plugin = DownloadDispatcher_Source_PluginFactory::create($plugin_name, $config, $log); $plugin->run(); diff --git a/source/lib/DownloadDispatcher/Sync/IPlugin.class.php b/source/lib/DownloadDispatcher/Sync/IPlugin.class.php new file mode 100644 index 0000000..b33918f --- /dev/null +++ b/source/lib/DownloadDispatcher/Sync/IPlugin.class.php @@ -0,0 +1,16 @@ + \ No newline at end of file diff --git a/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php b/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php new file mode 100644 index 0000000..e5e61b4 --- /dev/null +++ b/source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php @@ -0,0 +1,48 @@ +config = $config; + $this->log = $log; + $this->instance = $instance; + + $this->options = $this->config->get("sync.Rsync.{$this->instance}.options"); + $this->source = $this->config->get("sync.Rsync.{$this->instance}.source"); + $this->destination = $this->config->get("sync.Rsync.{$this->instance}.destination"); + } + + public function run() { + DownloadDispatcher_LogEntry::debug($this->log, "Running Rsync synchroniser: '{$this->instance}'"); + + $command = "/usr/bin/rsync {$this->options} '{$this->source}' '{$this->destination}'"; + DownloadDispatcher_LogEntry::debug($this->log, "Running foreground task: {$command}"); + + DownloadDispatcher_ForegroundTask::execute($command, null, null, null, array($this, 'output'), null, $this->instance); + } + + public function output($identifier, $data) { + DownloadDispatcher_LogEntry::debug($this->log, "{$identifier}: {$data}"); + } +} + +?> \ No newline at end of file diff --git a/source/lib/DownloadDispatcher/Sync/PluginFactory.class.php b/source/lib/DownloadDispatcher/Sync/PluginFactory.class.php new file mode 100644 index 0000000..47c1ca2 --- /dev/null +++ b/source/lib/DownloadDispatcher/Sync/PluginFactory.class.php @@ -0,0 +1,30 @@ + 'DownloadDispatcher/Sync/Plugin/', + ); + + + public static function init() { + + } + + public static function create($plugin, SihnonFramework_Config $config, SihnonFramework_Log $log, $instance) { + self::ensureScanned(); + + if (! self::isValidPlugin($plugin)) { + throw new Sihnon_Exception_InvalidPluginName($plugin); + } + + $classname = self::classname($plugin); + + return call_user_func(array($classname, 'create'), $config, $log, $instance); + } + +} + +?> \ No newline at end of file From 51ffefa5034a29578c19a9a93123f83ba9b61053 Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Tue, 13 Dec 2011 01:10:40 +0000 Subject: [PATCH 2/3] Add MediaFilesIterator Add a simple utility class that can be used to find media files (video and archives) from a directory tree. --- .../Utility/MediaFilesIterator.class.php | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php diff --git a/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php b/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php new file mode 100644 index 0000000..1ce7df2 --- /dev/null +++ b/source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php @@ -0,0 +1,9 @@ +current()->getFilename()); + } +} + +?> \ No newline at end of file From 0646848193010ef238f131e5745cba60fa757ec6 Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Tue, 13 Dec 2011 01:11:57 +0000 Subject: [PATCH 3/3] Update TV Source to find media files Updated TV Source to iterate over media files and check each against the cache to ignore previously seen items. --- .../Plugin/RouterboardFirmware.class.php | 2 +- .../Source/Plugin/TV.class.php | 26 ++++++++------- .../Source/PluginBase.class.php | 32 +++++++++++++++++++ 3 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 source/lib/DownloadDispatcher/Source/PluginBase.class.php diff --git a/source/lib/DownloadDispatcher/Source/Plugin/RouterboardFirmware.class.php b/source/lib/DownloadDispatcher/Source/Plugin/RouterboardFirmware.class.php index 178983c..5860909 100644 --- a/source/lib/DownloadDispatcher/Source/Plugin/RouterboardFirmware.class.php +++ b/source/lib/DownloadDispatcher/Source/Plugin/RouterboardFirmware.class.php @@ -1,6 +1,6 @@ config->get('sources.TV.input-directories'); foreach ($source_dirs as $dir) { if (is_dir($dir) && is_readable($dir)) { - $this->process_dir($dir); + $iterator = new DownloadDispatcher_Utility_MediaFilesIterator(new DownloadDispatcher_Utility_VisibleFilesIterator(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir)))); + foreach ($iterator as /** @var SplFileInfo */ $file) { + $this->process_matched_file($file->getPath(), $file->getFilename()); + } } else { DownloadDispatcher_LogEntry::warning($this->log, "TV input directory '{$dir}' does not exist or cannot be read."); } } } - - protected function process_dir($dir) { - // TODO - Iterate over the contents of the directory, process files and recurse deeper - } - + protected function process_matched_file($dir, $file) { // TODO - Handle movement of the matched file to the correct output directory // Handle direct media files, and also RAR archives + DownloadDispatcher_LogEntry::debug($this->log, "Media file: {$file}"); + + // Check to see if this file has been handled previously + if ($this->check_processed($dir . '/' . $file)) { + DownloadDispatcher_LogEntry::debug($this->log, "Skipping previously seen file"); + return; + } + } protected function identify_output_dir($dir, $file) { @@ -58,11 +65,6 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_PluginBase // TODO - use tvrenamer to update the filenames } - protected function mark_processed($dir, $file) { - // TODO - Update the cache to show that a file has already been handled - // TODO - Upstream caching - } - } ?> \ No newline at end of file diff --git a/source/lib/DownloadDispatcher/Source/PluginBase.class.php b/source/lib/DownloadDispatcher/Source/PluginBase.class.php new file mode 100644 index 0000000..72567c3 --- /dev/null +++ b/source/lib/DownloadDispatcher/Source/PluginBase.class.php @@ -0,0 +1,32 @@ +init_cache(); + + if ( ! in_array($file, static::$source_cache[get_called_class()])) { + static::$source_cache[get_called_class()][] = $file; + } + + // TODO - flush cache to persistent storage + } + + protected function check_processed($file) { + $this->init_cache(); + + return in_array($file, static::$source_cache[get_called_class()]); + } + +} + +?> \ No newline at end of file