ben/download-dispatcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
Ben Roberts
2012-01-10 01:37:13 +00:00
parent 0646848193 836febcfcc
commit 77ecf90796
9 changed files with 314 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@@ -0,0 +1,3 @@
[submodule "source/3rdparty/tvrenamer"]
path = source/3rdparty/tvrenamer
url = https://github.com/meermanr/TVSeriesRenamer.git

1
source/3rdparty/tvrenamer vendored Submodule

Submodule source/3rdparty/tvrenamer added at b1eb1ceceb

10
source/lib/DownloadDispatcher/Exceptions.class.php Normal file
View File

@@ -0,0 +1,10 @@
<?php
class DownloadDispatcher_Exception_SourcePluginException extends DownloadDispatcher_Exception {};
class DownloadDispatcher_Exception_PreviouslySeenContent extends DownloadDispatcher_Exception_SourcePluginException {};
class DownloadDispatcher_Exception_UnidentifiedContent extends DownloadDispatcher_Exception_SourcePluginException {};
class DownloadDispatcher_Exception_UnacceptableContent extends DownloadDispatcher_Exception_SourcePluginException {};
class DownloadDispatcher_Exception_DuplicateContent extends DownloadDispatcher_Exception_SourcePluginException {};
class DownloadDispatcher_Exception_UnprocesseableContent extends DownloadDispatcher_Exception_SourcePluginException {};
?>

25
source/lib/DownloadDispatcher/Main.class.php Normal file
View File

@@ -0,0 +1,25 @@
<?php
class DownloadDispatcher_Main extends SihnonFramework_Main {
protected $daemon;
public function __construct() {
parent::__construct();
}
public function init() {
parent::init();
try {
$this->daemon = new DownloadDispatcher_Daemon($this->config);
} catch (SihnonFramework_Exception_AlreadyRunning $e) {
DownloadDispatcher_LogEntry::error($this->log, "Another instance is already running, exiting this process now.");
exit(0);
}
}
}
?>

11
source/lib/DownloadDispatcher/Processor.class.php
View File

@@ -23,7 +23,7 @@ class DownloadDispatcher_Processor {
$plugin = DownloadDispatcher_Sync_PluginFactory::create($plugin_name, $config, $log, $instance);
$plugin->run();
} catch(SihnonFramework_Exception_LogException $e) {
} catch(SihnonFramework_Exception_PluginException $e) {
SihnonFramework_LogEntry::warning($log, $e->getMessage());
}
}
@@ -31,13 +31,14 @@ class DownloadDispatcher_Processor {
// Find the list of available source plugins
DownloadDispatcher_Source_PluginFactory::scan();
$source_plugins = DownloadDispatcher_Source_PluginFactory::getValidPlugins();
$enabled_plugins = $config->get('sources');
$source_plugins = $config->get('sources');
foreach ($source_plugins as $plugin_name) {
if (in_array($plugin_name, $enabled_plugins)) {
try {
$plugin = DownloadDispatcher_Source_PluginFactory::create($plugin_name, $config, $log);
$plugin->run();
} catch(DownloadDispatcher_Exception_PluginException $e) {
SihnonFramework_LogEntry::warning($log, $e->getMessage());
}
}
}

251
source/lib/DownloadDispatcher/Source/Plugin/TV.class.php
View File

@@ -7,11 +7,16 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug
*
* @var string
*/
const PLUGIN_NAME = "TV";
const PLUGIN_NAME = "TV";
protected $config;
protected $log;
protected $output_dir_cache;
protected $input_dirs;
protected $output_dir;
public static function create($config, $log) {
return new self($config, $log);
}
@@ -19,18 +24,20 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug
protected function __construct($config, $log) {
$this->config = $config;
$this->log = $log;
$this->input_dirs = $this->config->get('sources.TV.input');
$this->output_dir = $this->config->get('sources.TV.output');
}
public function run() {
DownloadDispatcher_LogEntry::debug($this->log, 'Running TV dispatcher');
// Iterate over source directories, and move matched files to the output directory
$source_dirs = $this->config->get('sources.TV.input-directories');
foreach ($source_dirs as $dir) {
foreach ($this->input_dirs as $dir) {
if (is_dir($dir) && is_readable($dir)) {
$iterator = new DownloadDispatcher_Utility_MediaFilesIterator(new DownloadDispatcher_Utility_VisibleFilesIterator(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir))));
foreach ($iterator as /** @var SplFileInfo */ $file) {
$this->process_matched_file($file->getPath(), $file->getFilename());
$this->processMatchedFile($file->getPath(), $file->getFilename(), $file->getExtension());
}
} else {
DownloadDispatcher_LogEntry::warning($this->log, "TV input directory '{$dir}' does not exist or cannot be read.");
@@ -38,33 +45,239 @@ class DownloadDispatcher_Source_Plugin_TV extends DownloadDispatcher_Source_Plug
}
}
protected function process_matched_file($dir, $file) {
protected function processMatchedFile($dir, $file, $type) {
// TODO - Handle movement of the matched file to the correct output directory
// Handle direct media files, and also RAR archives
DownloadDispatcher_LogEntry::debug($this->log, "Media file: {$file}");
DownloadDispatcher_LogEntry::debug($this->log, "Media file: '{$file}'.");
try {
// Check to see if this file has been handled previously
if ($this->check_processed($dir . '/' . $file)) {
DownloadDispatcher_LogEntry::debug($this->log, "Skipping previously seen file");
return;
// Check to see if this file has been handled previously
if ($this->checkProcessed($dir . '/' . $file)) {
throw new DownloadDispatcher_Exception_PreviouslySeenContent($file);
}
$full_output_dir = $this->identifyOutputDir($dir, $file);
$this->checkDuplicates($full_output_dir, $file);
$this->copyOutput($type, $dir, $file, $full_output_dir);
$this->renameOutput($full_output_dir);
// This file has been dealt with, so no need to look at it in subsequent operations
$this->markProcessed($file);
} catch (DownloadDispatcher_Exception_PreviouslySeenContent $e) {
DownloadDispatcher_LogEntry::debug($this->log, "Skipping previously seen file '{$e->getMessage()}'.");
} catch (DownloadDispatcher_Exception_UnidentifiedContent $e) {
DownloadDispatcher_LogEntry::warning($this->log, "TV output directory for '{$e->getMessage()}' could not be identified; you may need to create one.");
} catch (DownloadDispatcher_Exception_UnacceptableContent $e) {
DownloadDispatcher_LogEntry::warning($this->log, "Skipping '{$e->getMessage()}' due to dubious contents.");
// Forget the download upstream so a new copy can be fetched
$file = $e->getMessage();
$this->forgetDownload($this->normalise($file), $this->season($file), $this->episode($file));
} catch (DownloadDispatcher_Exception_DuplicateContent $e) {
DownloadDispatcher_LogEntry::info($this->log, "Skipping duplicate file '{$e->getMessage()}'.");
} catch (DownloadDispatcher_Exception_UnprocesseableContent $e) {
DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$e->getMessage()}' to the destination directory.");
}
}
protected function identifyOutputDir($dir, $file) {
// TODO - Generate the correct output directory, apply any special case mappings, and ensure the destination exists
if (is_null($this->output_dir_cache)) {
$this->scanOutputDir();
}
$normalised_file = $this->normalise($file);
if (array_key_exists($normalised_file, $this->output_dir_cache)) {
$season = $this->season($file);
$full_output_dir = "{$this->output_dir}/{$this->output_dir_cache[$normalised_file]}/Season {$season}";
if (is_dir($full_output_dir)) {
return $full_output_dir;
}
}
throw new DownloadDispatcher_Exception_UnidentifiedContent($file);
}
protected function identify_output_dir($dir, $file) {
// TODO - Generate the correct output directory, apply any special case mappings, and ensure the destination exists
protected function scanOutputDir() {
// Get a list of the series and season directories available in normalised form
DownloadDispatcher_LogEntry::debug($this->log, "Scanning TV output directory ({$this->output_dir})");
$this->output_dir_cache = array();
$series_iterator = new DownloadDispatcher_Utility_VisibleFilesIterator(new DirectoryIterator($this->output_dir));
foreach ($series_iterator as $series) {
$series_name = $series->getBasename();
$normalised_series = $this->normalise($series_name);
$this->output_dir_cache[$normalised_series] = $series_name;
}
}
protected function identify_duplicate($dir, $file) {
// TODO - Verify that the file we've found hasn't already been processed
// Use the cache to reduce processing overhead
// TODO - Upstream caching
protected function normalise($name) {
if (preg_match('/(.*?)([\s\.]us)?([\s\.]+(19|20)\d{2})?[\s\.]+(\d+x\d+|s\d+e\d+|\d{3,4}).*/i', $name, $matches)) {
$name = $matches[1];
}
$name = preg_replace('/[^a-zA-Z0-9]/', ' ', $name);
$name = preg_replace('/ +/', ' ', $name);
$name = strtolower($name);
$name = trim($name);
return $name;
}
protected function rename_output($dir, $file) {
// TODO - use tvrenamer to update the filenames
protected function season($name) {
$set_season = function($a) {
for ($i = 1, $l = count($a); $i < $l; ++$i) {
if ($a[$i]) {
return ltrim($a[$i], '0');
}
}
return null;
};
if (preg_match('/(\d+)x\d+|s(\d+)e\d+|(?:(?:19|20)\d{2}[\s\.]+)?(\d+)\d{2}/i', $name, $matches)) {
return $set_season($matches);
} else {
return 0;
}
}
protected function episode($name) {
$set_episode = function($a) {
for ($i = 1, $l = count($a); $i < $l; ++$i) {
if ($a[$i]) {
return ltrim($a[$i], '0');
}
}
return null;
};
if (preg_match('/\d+x(\d+)|s\d+e(\d+)|(?:(?:19|20)\d{2}[\s\.]+)?\d+(\d{2})/i', $name, $matches)) {
return $set_episode($matches);
} else {
return 0;
}
}
protected function checkDuplicates($dir, $file) {
$episode = $this->episode($file);
$iterator = new DownloadDispatcher_Utility_MediaFilesIterator(new DownloadDispatcher_Utility_VisibleFilesIterator(new DirectoryIterator($dir)));
foreach ($iterator as /** @var SplFileInfo */ $existing_file) {
$existing_episode = $this->episode($existing_file->getFilename());
if ($existing_episode == $episode) {
throw new DownloadDispatcher_Exception_DuplicateContent($file);
}
}
}
protected function copyOutput($type, $source_dir, $source_file, $destination_dir) {
switch (strtolower($type)) {
case 'rar': {
DownloadDispatcher_LogEntry::info($this->log, "Unrarring '{$source_file}' into '{$destination_dir}'.");
$safe_source_file = escapeshellarg("{$source_dir}/{$source_file}");
$command = "/usr/bin/unrar e -p- -sm8192 -y {$safe_source_file}";
DownloadDispatcher_LogEntry::debug($this->log, "Unrarring '{$source_file}' with command: {$command}");
list ($code,$output,$error) = DownloadDispatcher_ForegroundTask::execute($command, $destination_dir);
if ($code == 3) {
throw new DownloadDispatcher_Exception_UnacceptableContent($source_file);
} else if ($code != 0) {
DownloadDispatcher_LogEntry::warning($this->log, "Failed to unrar '{$source_dir}/{$source_file}'.");
throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file);
}
} break;
case 'avi': {
// Verify that the file isn't a fake
$safe_source_file = escapeshellarg($source_file);
$command = "file {$safe_source_file}";
DownloadDispatcher_LogEntry::debug($this->log, "Verifying '{$source_file}' contents with command: {$command}");
list($code, $output, $error) = DownloadDispatcher_ForegroundTask::execute($command, $source_dir);
if ($code != 0) {
DownloadDispatcher_LogEntry::warning($this->log, "Failed to determine contents of '{$source_dir}/{$source_file}'.");
throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file);
}
if (preg_match('/Microsoft ASF/', $output)) {
throw new DownloadDispatcher_Exception_UnacceptableContent($source_file);
}
} // continue into the next case
default: {
DownloadDispatcher_LogEntry::info($this->log, "Copying '{$source_file}' to '{$destination_dir}'.");
$result = copy("{$source_dir}/{$source_file}", "{$destination_dir}/{$source_file}");
if ( ! $result) {
DownloadDispatcher_LogEntry::warning($this->log, "Failed to copy '{$source_dir}/{$source_file}' to output directory '{$destination_dir}'.");
throw new DownloadDispatcher_Exception_UnprocesseableContent($source_file);
}
}
}
return true;
}
protected function renameOutput($dir) {
$cwd = getcwd();
$command = <<<EOSH
{$cwd}/3rdparty/tvrenamer/tvrenamer.pl \
--include_series \
--nogroup \
--pad=2 \
--scheme=XxYY \
--preproc='s/x264//;' \
--postproc='s/(?:-+img|-+a).*(\.[a-zA-Z0-9]+$)/\1/;' \
--unattended \
--dubious \
--cache
EOSH;
DownloadDispatcher_LogEntry::debug($this->log, "Executing tvrenamer command in '{$dir}': {$command}");
DownloadDispatcher_ForegroundTask::execute($command, $dir);
}
protected function forgetDownload($series, $season, $episode) {
$base_url = $this->config->get('sources.TV.flexget-url');
$username = $this->config->get('sources.TV.flexget-username');
$password = $this->config->get('sources.TV.flexget-password');
$url = "{$base_url}execute/";
$data = array(
'options' => "--series-forget '{$series}' 's{$season}e{$episode}'",
'submit' => 'Start Execution',
);
DownloadDispatcher_LogEntry::debug($this->log, "Sending flexget series-forget command to {$url} with options '{$data['options']}'.");
$request = new HttpRequest($url, HTTP_METH_POST, array(
'httpauth' => "{$username}:{$password}",
'httpauthtype' => HTTP_AUTH_BASIC,
));
$request->setPostFields($data);
$response = $request->send();
DownloadDispatcher_LogEntry::debug($this->log, "Response code: {$response->getResponseCode()}.");
if ($response->getResponseCode() == 200) {
DownloadDispatcher_LogEntry::info($this->log, "Successfully made flexget forget about {$series} s{$season}e{$episode}.");
} else {
DownloadDispatcher_LogEntry::warning($this->log, "Failed to make flexget forget about {$series} s{$season}e{$episode}.");
}
}
}
?>
?>

31
source/lib/DownloadDispatcher/Source/PluginBase.class.php
View File

@@ -2,27 +2,42 @@
class DownloadDispatcher_Source_PluginBase extends DownloadDispatcher_PluginBase {
static protected $source_cache = array();
static protected $cache;
protected function init_cache() {
static protected $source_cache = null;
static protected $source_cache_file = 'source_cache';
static protected $cache_lifetime = 86400;
protected function initSourceCache() {
if ( ! static::$cache) {
static::$cache = DownloadDispatcher_Main::instance()->cache();
}
if (is_null(static::$source_cache)) {
try {
static::$source_cache = unserialize(static::$cache->fetch(static::$source_cache_file, static::$cache_lifetime));
} catch (SihnonFramework_Exception_CacheObjectNotFound $e) {
static::$source_cache = array();
}
}
if ( ! array_key_exists(get_called_class(), static::$source_cache)) {
// TODO - attempt to load data from persistent storage
static::$source_cache[get_called_class()] = array();
}
}
protected function mark_processed($file) {
$this->init_cache();
protected function markProcessed($file) {
$this->initSourceCache();
if ( ! in_array($file, static::$source_cache[get_called_class()])) {
static::$source_cache[get_called_class()][] = $file;
}
// TODO - flush cache to persistent storage
static::$cache->store(static::$source_cache_file, serialize(static::$source_cache));
}
protected function check_processed($file) {
$this->init_cache();
protected function checkProcessed($file) {
$this->initSourceCache();
return in_array($file, static::$source_cache[get_called_class()]);
}

6
source/lib/DownloadDispatcher/Sync/Plugin/Rsync.class.php
View File

@@ -13,6 +13,7 @@ class DownloadDispatcher_Sync_Plugin_Rsync extends DownloadDispatcher_PluginBase
protected $log;
protected $instance;
protected $enabled;
protected $options;
protected $source;
protected $destination;
@@ -26,12 +27,17 @@ class DownloadDispatcher_Sync_Plugin_Rsync extends DownloadDispatcher_PluginBase
$this->log = $log;
$this->instance = $instance;
$this->enabled = $this->config->get("sync.Rsync.{$this->instance}.enabled", true);
$this->options = $this->config->get("sync.Rsync.{$this->instance}.options");
$this->source = $this->config->get("sync.Rsync.{$this->instance}.source");
$this->destination = $this->config->get("sync.Rsync.{$this->instance}.destination");
}
public function run() {
if ( ! $this->enabled) {
return;
}
DownloadDispatcher_LogEntry::debug($this->log, "Running Rsync synchroniser: '{$this->instance}'");
$command = "/usr/bin/rsync {$this->options} '{$this->source}' '{$this->destination}'";

10
source/lib/DownloadDispatcher/Utility/MediaFilesIterator.class.php
View File

@@ -1,8 +1,14 @@
<?php
class DownloadDispatcher_Utility_MediaFilesIterator extends FilterIterator {
public function accept() {
return preg_match('/(?<!\.sample)\.(?:avi|ogm|m4v|mkv|mov|mp4|mpg|srt|rar)$/i', $this->current()->getFilename());
public function accept() {
$filename = $this->current()->getFilename();
if (preg_match('/^sample/', $filename)) {
return false;
}
if (preg_match('/(?<!(?:\.|-)sample)\.(?:avi|ogm|m4v|mkv|mov|mp4|mpg|srt|rar)$/i', $filename)) {
return true;
}
}
}