ben/handbrake-cluster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Latest version with support for webui and database logging

Browse Source
This commit is contained in:
Ben Roberts
2010-06-08 23:13:36 +01:00
parent 9358c100ae
commit bfcb32f9cb
3 changed files with 160 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
handbrake-cluster-client.pl
View File

@@ -1,12 +1,14 @@
#!/usr/bin/perl
package HandbrakeCluster::Client;
use warnings;
use strict;
use Data::Dumper;
use Gearman::Client;
use Getopt::Long;
use Log::Handler;
use Log::Log4perl;
use MIME::Lite::TT::HTML;
use Pod::Usage;
use Storable qw/freeze thaw/;
@@ -14,15 +16,12 @@ use YAML::Any;
# Globals
my $default_options = {
verbose => 0,
debug => 0,
quiet => 0,
log_file => '/tmp/handbrake-cluster-client.log',
silent => 0,
log_config => 'logging.conf',
help => 0,
pretend => 0,
config_file => '',
job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
gearman_prefix => '',
limit => [],
email_target => '',
email_sender => '',
@@ -55,14 +54,11 @@ my $rip_options = { map { $_ => undef } keys %$default_rip_options };
Getopt::Long::Configure( qw(bundling no_getopt_compat) );
GetOptions(
'verbose|v+' => \$options->{verbose},
'debug|d' => \$options->{debug},
'quiet|q' => \$options->{quiet},
'silent|s' => \$options->{silent},
'log|l=s' => \$options->{log_file},
'log-config|l=s' => \$options->{log_config},
'help|h' => \$options->{help},
'pretend|n' => \$options->{pretend},
'job-servers|j=s@' => \$options->{job_servers},
'gearman-prefix=s' => \$options->{gearman_prefix},
'config|c=s' => \$options->{config_file},
'limit|L=s' => sub { my ($name, $value) = @_; $options->{limit} = [split(/,/, $value)]; },
'nice|N=i' => \$rip_options->{nice},
@@ -95,33 +91,14 @@ $jobs{__commandline} = $rip_options if $options->{title};
die "No rips specified" unless %jobs;
# Setup logging
my $log = Log::Handler->new();
# default to logging warning+, unless quiet in which case log error+, or debug in which case log everything
my $maxLogLevel = ($options->{debug} ? 'debug' : ($options->{quiet} ? 3 : $options->{verbose} + 4));
# Ignore the quiet option when logging to disk
my $maxFileLogLevel = ($options->{debug} ? 'debug' : 'info');
Log::Log4perl->init($options->{log_config});
my $client_log = Log::Log4perl->get_logger('HandbrakeCluster::Client');
$client_log->debug("Logging started");
if ( ! $options->{silent}) {
$log->add(
screen => {
log_to => 'STDOUT',
minlevel => 'emergency',
maxlevel => $maxLogLevel,
},
);
}
if ( $options->{log_file}) {
$log->add(
file => {
filename => $options->{log_file},
minlevel => 'emergency',
maxlevel => $maxFileLogLevel,
},
);
}
# Setup the distribution client
my $client = Gearman::Client->new;
$client->prefix($options->{gearman_prefix}) if $options->{gearman_prefix};
$client->job_servers($options->{job_servers});
# Add new ripping task for each job to run
@@ -131,7 +108,13 @@ foreach my $job_id (keys %jobs) {
my $job = $jobs{$job_id};
# Check that the job hasn't been restricted by the limit option
next if (!$options->{limit} || !$options->{job_permitted}->{$job_id});
if (@{ $options->{limit} } && !$options->{job_permitted}->{$job_id}) {
$client_log->info("Skipped job $job_id due to limit restrictions");
next;
}
# Create a new database record for this job, and grab the new id
$job->{db_job_id} = 43;
$progress{$job_id} = 0;
@@ -148,14 +131,14 @@ foreach my $job_id (keys %jobs) {
on_complete => \&on_complete_handler,
on_retry => sub {
my $attempt = shift or die;
$log->warning("Retrying '$job_id' rip");
$client_log->warning("Retrying '$job_id' rip");
},
on_fail => sub {
$log->warning("Rip '$job_id' failed");
$client_log->warning("Rip '$job_id' failed");
},
}
);
$log->info("Enqueued '$job_id' rip");
$client_log->info("Enqueued '$job_id' rip");
}
$taskset->wait;
@@ -188,7 +171,7 @@ sub on_complete_handler {
$email->send;
}
$log->notice("Completed rip to $response->{real_output_filename}");
$client_log->info("Completed rip to $response->{real_output_filename}");
}
sub display_progress {

156
handbrake-cluster-worker.pl
View File

@@ -1,5 +1,7 @@
#!/usr/bin/perl
package HandbrakeCluster::Worker;
use warnings;
use strict;
@@ -9,75 +11,70 @@ use Getopt::Long;
use IO::Select;
use IO::Socket;
use IPC::Open3;
use Log::Handler;
use Log::Log4perl;
use PHP::Serialization;
use Pod::Usage;
use String::Random qw/random_string/;
use Storable qw/freeze thaw/;
use Switch;
use Symbol qw/gensym/;
use YAML::Any;
# Globals
our %options = (
verbose => 0,
quiet => 0,
log_file => '/tmp/handbrake-cluster-worker.log',
silent => 0,
our $default_options = {
log_config => 'logging.conf',
help => 0,
pretend => 0,
job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
gearman_prefix => '',
handbrake => '/usr/bin/HandBrakeCLI',
);
config_file => '',
php_client => 0,
};
my $options = { map { $_ => undef } keys %$default_options };
Getopt::Long::Configure( qw(bundling no_getopt_compat) );
GetOptions(
'help|h' => \$options{help},
'verbose|v' => \$options{verbose},
'debug|d' => \$options{debug},
'quiet|q' => \$options{quiet},
'silent|s' => \$options{silent},
'log|l=s' => \$options{log_file},
'pretend|n' => \$options{pretend},
'job-servers|j=s@' => \$options{job_servers},
'handbrake' => \$options{handbrake},
'help|h' => \$options->{help},
'log-config|l=s' => \$options->{log_config},
'pretend|n' => \$options->{pretend},
'job-servers|j=s@' => \$options->{job_servers},
'gearman-prefix=s' => \$options->{gearman_prefix},
'handbrake' => \$options->{handbrake},
'config|c=s' => \$options->{config_file},
) or pod2usage(-verbose => 0);
pod2usage(-verbose => 1) if ($options{help});
pod2usage(-verbose => 1) if ($options->{help});
# Parse the configuration file (if any), and merge/validate the options
my $config = parse_config($options->{config_file});
($config, $options) = process_config($config, $options, $default_options);
# Setup logging
my $log = Log::Handler->new();
my $maxLogLevel = ($options{debug} ? 'debug' : ($options{quiet} ? 3 : $options{verbose} + 4)); # default to logging warning+, unless quiet in which case log error+, or debug in which case log everything
my $maxFileLogLevel = ($options{debug} ? 'debug' : 'info');
if ( ! $options{silent}) {
$log->add(
screen => {
log_to => 'STDOUT',
minlevel => 'emergency',
maxlevel => $maxLogLevel,
},
);
}
if ( $options{log_file}) {
$log->add(
file => {
filename => $options{log_file},
minlevel => 'emergency',
maxlevel => $maxFileLogLevel,
},
);
}
Log::Log4perl->init($options->{log_config});
my $worker_log = Log::Log4perl->get_logger('HandbrakeCluster::Worker');
$worker_log->debug("Logging started");
# Setup the worker, and listen for jobs
my $worker = Gearman::Worker->new(job_servers => $options{job_servers});
my $worker = Gearman::Worker->new(job_servers => $options->{job_servers});
$worker->prefix($options->{gearman_prefix}) if $options->{gearman_prefix};
$worker->register_function(handbrake_rip => \&handbrake_rip);
$worker->work while 1;
sub handbrake_rip {
my $job = shift;
my $rip_options = thaw($job->arg);
my $rip_options;
if ($options->{php_client}) {
$rip_options = PHP::Serialization::unserialize($job->arg);
} else {
$rip_options = thaw($job->arg);
}
my $response = {};
$log->notice("Beginning new rip to $rip_options->{output_filename}");
# Setup logging for this job
my $job_log = Log::Log4perl->get_logger('HandbrakeCluster::Worker::Job');
Log::Log4perl::MDC->put('job_id', $rip_options->{db_job_id});
$job_log->info("Beginning new rip to $rip_options->{output_filename}");
# Generate a unique filename based on the output filename to prevent clashes from previous runs
my $uuid = random_string('cccccc');
@@ -91,7 +88,7 @@ sub handbrake_rip {
'nice', '-n', $rip_options->{nice},
# Construct the handbrake command line
$options{handbrake},
$options->{handbrake},
get_options($rip_options, 'input_filename', '-i'),
get_options($rip_options, 'real_output_filename', '-o'),
get_options($rip_options, 'title'),
@@ -114,7 +111,7 @@ sub handbrake_rip {
$job->set_status(0, 100);
# Execute the ripping process
$log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd));
$job_log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd));
my ($child_in, $child_out, $child_err) = map gensym, 1..3;
my $child_pid = open3($child_in, $child_out, $child_err, @handbrake_cmd);
# No need to write to the child process
@@ -135,11 +132,11 @@ sub handbrake_rip {
foreach my $handle (@ready) {
my $bytes_read = sysread($handle, my $buf = '', 1024);
if ($bytes_read == -1) {
$log->error("Error reading from HandBrake socket: $!");
$job_log->error("Error reading from HandBrake socket: $!");
$select->remove($child_out);
next;
} elsif ($bytes_read == 0) {
$log->debug("HandBrake socket closed");
$job_log->debug("HandBrake socket closed");
$select->remove($handle);
next;
}
@@ -153,8 +150,8 @@ sub handbrake_rip {
$child_out_buffer = $lines[1];
if ($line =~ m/Encoding: task \d+ of \d+, (\d+\.\d+) %/) {
my $numerator = $1;
$job->set_status($numerator, 1);
my $numerator = 100 * $1;
$job->set_status($numerator, 100);
}
}
} else {
@@ -166,7 +163,7 @@ sub handbrake_rip {
$child_err_buffer = $lines[1];
push @{ $response->{log} }, $line;
$log->notice($line);
$job_log->info($line);
}
}
}
@@ -182,9 +179,9 @@ sub handbrake_rip {
$response->{status} = $? >> 8;
$response->{success} = $response->{status} == 0;
if ($response->{success}) {
$log->notice("Finished rip to $response->{real_output_filename}");
$job_log->info("Finished rip to $response->{real_output_filename}");
} else {
$log->warning("Ripping process returned error status: $response->{status}");
$job_log->warning("Ripping process returned error status: $response->{status}");
}
return freeze($response);
@@ -217,6 +214,37 @@ sub get_options {
}
}
# Reads configuration options from a config file, expands the internal references, and returns the expanded form.
sub parse_config {
my $config_file = shift;
return {} unless defined $config_file;
my $config = YAML::Any::LoadFile($options->{config_file}) or die 'Unable to load configuration file: ' . $!;
return $config;
}
sub process_config {
my $config = shift or die;
my $options = shift or die;
my $default_options = shift or die;
# Update any unset options with values from config file
foreach my $option (keys %$options) {
# Update the value of any option which has not been set by a command line argument
if (!$options->{$option}) {
# Try the config file first, otherwise use the default value
if (defined $config->{options}->{$option}) {
$options->{$option} = $config->{options}->{$option};
} else {
$options->{$option} = $default_options->{$option};
}
}
}
return ($config, $options);
}
__END__;
=head1 NAME
@@ -225,9 +253,7 @@ __END__;
=head1 SYNOPSIS
handbrake-cluster-worker.pl -h|--help
handbrake-cluster-worker.pl [[-v|--verbose] | [-d|--debug] |
[-q|--quiet] | [-s|--silent]]
[-l|--log <log file>]
handbrake-cluster-worker.pl [-l|--log-config <log4perl config file>]
[-n|--pretend]
[-j|--job-servers <server list>]
[--handbrake <handbrake executable>]
@@ -246,26 +272,10 @@ job servers to use are configurable by command line arguments.
Displays this help information and quits.
=item B<-v|--verbose>
=item B<-l|--log-config E<lt>log4perl config fileE<gt>>
Displays verbose logging information.
=item B<-d|--debug>
Displays full debugging information, including all output from HandBrake.
=item B<-q|--quiet>
Displays only errors.
=item B<-s|--silent>
Displays no output whatsoever, useful for scripting. Output will still be
logged to disk if a file is specified.
=item B<-l|--log E<lt>log fileE<gt>>
Specifies the name of a file to write logging information to.
Specifies the name of a Log4perl configuration file to control logging.
This file is expected to define a logger named HandbrakeCluster::Worker.
=item B<-n|--pretend>

45
worker.yml.dist Normal file
View File

@@ -0,0 +1,45 @@
---
options:
debug: 1
log_file: /tmp/handbrake-cluster-client.log
job_servers:
- build0.example.com
- build1.example.com
gearman_prefix: dev
log_database:
driver: mysql
host: localhost
database: handbrake_cluster
user: handbrake
password:
table: worker_log
columns:
- level
- ctime
- pid
- hostname
- progname
- line
- message
values:
- %level
- %time
- %pid
- %hostname
- %progname
- %line
- %message
message_pattern:
- %L
- %T
- %P
- %H
- %S
- %l
- %m
message_layout: %m
timeformat: %s
# vim: set ft=yaml ts=2 expandtab :