ben/handbrake-cluster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Latest version with support for webui and database logging

Browse Source
This commit is contained in:
Ben Roberts
2010-06-08 23:13:36 +01:00
parent 9358c100ae
commit bfcb32f9cb
3 changed files with 160 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
handbrake-cluster-worker.pl
View File

@@ -1,5 +1,7 @@
#!/usr/bin/perl
package HandbrakeCluster::Worker;
use warnings;
use strict;
@@ -9,75 +11,70 @@ use Getopt::Long;
use IO::Select;
use IO::Socket;
use IPC::Open3;
use Log::Handler;
use Log::Log4perl;
use PHP::Serialization;
use Pod::Usage;
use String::Random qw/random_string/;
use Storable qw/freeze thaw/;
use Switch;
use Symbol qw/gensym/;
use YAML::Any;
# Globals
our %options = (
verbose => 0,
quiet => 0,
log_file => '/tmp/handbrake-cluster-worker.log',
silent => 0,
help => 0,
pretend => 0,
job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
handbrake => '/usr/bin/HandBrakeCLI',
);
our $default_options = {
log_config => 'logging.conf',
help => 0,
pretend => 0,
job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
gearman_prefix => '',
handbrake => '/usr/bin/HandBrakeCLI',
config_file => '',
php_client => 0,
};
my $options = { map { $_ => undef } keys %$default_options };
Getopt::Long::Configure( qw(bundling no_getopt_compat) );
GetOptions(
'help|h' => \$options{help},
'verbose|v' => \$options{verbose},
'debug|d' => \$options{debug},
'quiet|q' => \$options{quiet},
'silent|s' => \$options{silent},
'log|l=s' => \$options{log_file},
'pretend|n' => \$options{pretend},
'job-servers|j=s@' => \$options{job_servers},
'handbrake' => \$options{handbrake},
'help|h' => \$options->{help},
'log-config|l=s' => \$options->{log_config},
'pretend|n' => \$options->{pretend},
'job-servers|j=s@' => \$options->{job_servers},
'gearman-prefix=s' => \$options->{gearman_prefix},
'handbrake' => \$options->{handbrake},
'config|c=s' => \$options->{config_file},
) or pod2usage(-verbose => 0);
pod2usage(-verbose => 1) if ($options{help});
pod2usage(-verbose => 1) if ($options->{help});
# Parse the configuration file (if any), and merge/validate the options
my $config = parse_config($options->{config_file});
($config, $options) = process_config($config, $options, $default_options);
# Setup logging
my $log = Log::Handler->new();
my $maxLogLevel = ($options{debug} ? 'debug' : ($options{quiet} ? 3 : $options{verbose} + 4)); # default to logging warning+, unless quiet in which case log error+, or debug in which case log everything
my $maxFileLogLevel = ($options{debug} ? 'debug' : 'info');
if ( ! $options{silent}) {
$log->add(
screen => {
log_to => 'STDOUT',
minlevel => 'emergency',
maxlevel => $maxLogLevel,
},
);
}
if ( $options{log_file}) {
$log->add(
file => {
filename => $options{log_file},
minlevel => 'emergency',
maxlevel => $maxFileLogLevel,
},
);
}
Log::Log4perl->init($options->{log_config});
my $worker_log = Log::Log4perl->get_logger('HandbrakeCluster::Worker');
$worker_log->debug("Logging started");
# Setup the worker, and listen for jobs
my $worker = Gearman::Worker->new(job_servers => $options{job_servers});
my $worker = Gearman::Worker->new(job_servers => $options->{job_servers});
$worker->prefix($options->{gearman_prefix}) if $options->{gearman_prefix};
$worker->register_function(handbrake_rip => \&handbrake_rip);
$worker->work while 1;
sub handbrake_rip {
my $job = shift;
my $rip_options = thaw($job->arg);
my $rip_options;
if ($options->{php_client}) {
$rip_options = PHP::Serialization::unserialize($job->arg);
} else {
$rip_options = thaw($job->arg);
}
my $response = {};
$log->notice("Beginning new rip to $rip_options->{output_filename}");
# Setup logging for this job
my $job_log = Log::Log4perl->get_logger('HandbrakeCluster::Worker::Job');
Log::Log4perl::MDC->put('job_id', $rip_options->{db_job_id});
$job_log->info("Beginning new rip to $rip_options->{output_filename}");
# Generate a unique filename based on the output filename to prevent clashes from previous runs
my $uuid = random_string('cccccc');
@@ -91,7 +88,7 @@ sub handbrake_rip {
'nice', '-n', $rip_options->{nice},
# Construct the handbrake command line
$options{handbrake},
$options->{handbrake},
get_options($rip_options, 'input_filename', '-i'),
get_options($rip_options, 'real_output_filename', '-o'),
get_options($rip_options, 'title'),
@@ -114,7 +111,7 @@ sub handbrake_rip {
$job->set_status(0, 100);
# Execute the ripping process
$log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd));
$job_log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd));
my ($child_in, $child_out, $child_err) = map gensym, 1..3;
my $child_pid = open3($child_in, $child_out, $child_err, @handbrake_cmd);
# No need to write to the child process
@@ -135,11 +132,11 @@ sub handbrake_rip {
foreach my $handle (@ready) {
my $bytes_read = sysread($handle, my $buf = '', 1024);
if ($bytes_read == -1) {
$log->error("Error reading from HandBrake socket: $!");
$job_log->error("Error reading from HandBrake socket: $!");
$select->remove($child_out);
next;
} elsif ($bytes_read == 0) {
$log->debug("HandBrake socket closed");
$job_log->debug("HandBrake socket closed");
$select->remove($handle);
next;
}
@@ -153,8 +150,8 @@ sub handbrake_rip {
$child_out_buffer = $lines[1];
if ($line =~ m/Encoding: task \d+ of \d+, (\d+\.\d+) %/) {
my $numerator = $1;
$job->set_status($numerator, 1);
my $numerator = 100 * $1;
$job->set_status($numerator, 100);
}
}
} else {
@@ -166,7 +163,7 @@ sub handbrake_rip {
$child_err_buffer = $lines[1];
push @{ $response->{log} }, $line;
$log->notice($line);
$job_log->info($line);
}
}
}
@@ -182,9 +179,9 @@ sub handbrake_rip {
$response->{status} = $? >> 8;
$response->{success} = $response->{status} == 0;
if ($response->{success}) {
$log->notice("Finished rip to $response->{real_output_filename}");
$job_log->info("Finished rip to $response->{real_output_filename}");
} else {
$log->warning("Ripping process returned error status: $response->{status}");
$job_log->warning("Ripping process returned error status: $response->{status}");
}
return freeze($response);
@@ -217,6 +214,37 @@ sub get_options {
}
}
# Reads configuration options from a config file, expands the internal references, and returns the expanded form.
sub parse_config {
my $config_file = shift;
return {} unless defined $config_file;
my $config = YAML::Any::LoadFile($options->{config_file}) or die 'Unable to load configuration file: ' . $!;
return $config;
}
sub process_config {
my $config = shift or die;
my $options = shift or die;
my $default_options = shift or die;
# Update any unset options with values from config file
foreach my $option (keys %$options) {
# Update the value of any option which has not been set by a command line argument
if (!$options->{$option}) {
# Try the config file first, otherwise use the default value
if (defined $config->{options}->{$option}) {
$options->{$option} = $config->{options}->{$option};
} else {
$options->{$option} = $default_options->{$option};
}
}
}
return ($config, $options);
}
__END__;
=head1 NAME
@@ -225,9 +253,7 @@ __END__;
=head1 SYNOPSIS
handbrake-cluster-worker.pl -h|--help
handbrake-cluster-worker.pl [[-v|--verbose] | [-d|--debug] |
[-q|--quiet] | [-s|--silent]]
[-l|--log <log file>]
handbrake-cluster-worker.pl [-l|--log-config <log4perl config file>]
[-n|--pretend]
[-j|--job-servers <server list>]
[--handbrake <handbrake executable>]
@@ -246,26 +272,10 @@ job servers to use are configurable by command line arguments.
Displays this help information and quits.
=item B<-v|--verbose>
=item B<-l|--log-config E<lt>log4perl config fileE<gt>>
Displays verbose logging information.
=item B<-d|--debug>
Displays full debugging information, including all output from HandBrake.
=item B<-q|--quiet>
Displays only errors.
=item B<-s|--silent>
Displays no output whatsoever, useful for scripting. Output will still be
logged to disk if a file is specified.
=item B<-l|--log E<lt>log fileE<gt>>
Specifies the name of a file to write logging information to.
Specifies the name of a Log4perl configuration file to control logging.
This file is expected to define a logger named HandbrakeCluster::Worker.
=item B<-n|--pretend>