From bfcb32f9cbf38165343bd4ee047b0c07e5d735be Mon Sep 17 00:00:00 2001 From: Ben Roberts Date: Tue, 8 Jun 2010 23:13:36 +0100 Subject: [PATCH] Latest version with support for webui and database logging --- handbrake-cluster-client.pl | 73 ++++++---------- handbrake-cluster-worker.pl | 164 +++++++++++++++++++----------------- worker.yml.dist | 45 ++++++++++ 3 files changed, 160 insertions(+), 122 deletions(-) create mode 100644 worker.yml.dist diff --git a/handbrake-cluster-client.pl b/handbrake-cluster-client.pl index 345123c..6192305 100755 --- a/handbrake-cluster-client.pl +++ b/handbrake-cluster-client.pl @@ -1,12 +1,14 @@ #!/usr/bin/perl +package HandbrakeCluster::Client; + use warnings; use strict; use Data::Dumper; use Gearman::Client; use Getopt::Long; -use Log::Handler; +use Log::Log4perl; use MIME::Lite::TT::HTML; use Pod::Usage; use Storable qw/freeze thaw/; @@ -14,22 +16,19 @@ use YAML::Any; # Globals my $default_options = { - verbose => 0, - debug => 0, - quiet => 0, - log_file => '/tmp/handbrake-cluster-client.log', - silent => 0, + log_config => 'logging.conf', help => 0, pretend => 0, config_file => '', job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'], + gearman_prefix => '', limit => [], - email_target => '', - email_sender => '', - email_subject => 'Rip completed', - email_timezone => 'Europe/London', - email_html => 'rip-completed.html', - email_text => 'rip-completed.txt', + email_target => '', + email_sender => '', + email_subject => 'Rip completed', + email_timezone => 'Europe/London', + email_html => 'rip-completed.html', + email_text => 'rip-completed.txt', }; my $options = { map { $_ => undef } keys %$default_options }; @@ -55,14 +54,11 @@ my $rip_options = { map { $_ => undef } keys %$default_rip_options }; Getopt::Long::Configure( qw(bundling no_getopt_compat) ); GetOptions( - 'verbose|v+' => \$options->{verbose}, - 'debug|d' => \$options->{debug}, - 'quiet|q' => \$options->{quiet}, - 'silent|s' => \$options->{silent}, - 'log|l=s' => \$options->{log_file}, + 'log-config|l=s' => \$options->{log_config}, 'help|h' => \$options->{help}, 'pretend|n' => \$options->{pretend}, 'job-servers|j=s@' => \$options->{job_servers}, + 'gearman-prefix=s' => \$options->{gearman_prefix}, 'config|c=s' => \$options->{config_file}, 'limit|L=s' => sub { my ($name, $value) = @_; $options->{limit} = [split(/,/, $value)]; }, 'nice|N=i' => \$rip_options->{nice}, @@ -95,33 +91,14 @@ $jobs{__commandline} = $rip_options if $options->{title}; die "No rips specified" unless %jobs; # Setup logging -my $log = Log::Handler->new(); -# default to logging warning+, unless quiet in which case log error+, or debug in which case log everything -my $maxLogLevel = ($options->{debug} ? 'debug' : ($options->{quiet} ? 3 : $options->{verbose} + 4)); -# Ignore the quiet option when logging to disk -my $maxFileLogLevel = ($options->{debug} ? 'debug' : 'info'); +Log::Log4perl->init($options->{log_config}); +my $client_log = Log::Log4perl->get_logger('HandbrakeCluster::Client'); +$client_log->debug("Logging started"); -if ( ! $options->{silent}) { - $log->add( - screen => { - log_to => 'STDOUT', - minlevel => 'emergency', - maxlevel => $maxLogLevel, - }, - ); -} -if ( $options->{log_file}) { - $log->add( - file => { - filename => $options->{log_file}, - minlevel => 'emergency', - maxlevel => $maxFileLogLevel, - }, - ); -} # Setup the distribution client my $client = Gearman::Client->new; +$client->prefix($options->{gearman_prefix}) if $options->{gearman_prefix}; $client->job_servers($options->{job_servers}); # Add new ripping task for each job to run @@ -131,7 +108,13 @@ foreach my $job_id (keys %jobs) { my $job = $jobs{$job_id}; # Check that the job hasn't been restricted by the limit option - next if (!$options->{limit} || !$options->{job_permitted}->{$job_id}); + if (@{ $options->{limit} } && !$options->{job_permitted}->{$job_id}) { + $client_log->info("Skipped job $job_id due to limit restrictions"); + next; + } + + # Create a new database record for this job, and grab the new id + $job->{db_job_id} = 43; $progress{$job_id} = 0; @@ -148,14 +131,14 @@ foreach my $job_id (keys %jobs) { on_complete => \&on_complete_handler, on_retry => sub { my $attempt = shift or die; - $log->warning("Retrying '$job_id' rip"); + $client_log->warning("Retrying '$job_id' rip"); }, on_fail => sub { - $log->warning("Rip '$job_id' failed"); + $client_log->warning("Rip '$job_id' failed"); }, } ); - $log->info("Enqueued '$job_id' rip"); + $client_log->info("Enqueued '$job_id' rip"); } $taskset->wait; @@ -188,7 +171,7 @@ sub on_complete_handler { $email->send; } - $log->notice("Completed rip to $response->{real_output_filename}"); + $client_log->info("Completed rip to $response->{real_output_filename}"); } sub display_progress { diff --git a/handbrake-cluster-worker.pl b/handbrake-cluster-worker.pl index 93fb365..53c2ce7 100755 --- a/handbrake-cluster-worker.pl +++ b/handbrake-cluster-worker.pl @@ -1,5 +1,7 @@ #!/usr/bin/perl +package HandbrakeCluster::Worker; + use warnings; use strict; @@ -9,75 +11,70 @@ use Getopt::Long; use IO::Select; use IO::Socket; use IPC::Open3; -use Log::Handler; +use Log::Log4perl; +use PHP::Serialization; use Pod::Usage; use String::Random qw/random_string/; use Storable qw/freeze thaw/; use Switch; use Symbol qw/gensym/; +use YAML::Any; # Globals -our %options = ( - verbose => 0, - quiet => 0, - log_file => '/tmp/handbrake-cluster-worker.log', - silent => 0, - help => 0, - pretend => 0, - job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'], - handbrake => '/usr/bin/HandBrakeCLI', -); +our $default_options = { + log_config => 'logging.conf', + help => 0, + pretend => 0, + job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'], + gearman_prefix => '', + handbrake => '/usr/bin/HandBrakeCLI', + config_file => '', + php_client => 0, +}; +my $options = { map { $_ => undef } keys %$default_options }; Getopt::Long::Configure( qw(bundling no_getopt_compat) ); GetOptions( - 'help|h' => \$options{help}, - 'verbose|v' => \$options{verbose}, - 'debug|d' => \$options{debug}, - 'quiet|q' => \$options{quiet}, - 'silent|s' => \$options{silent}, - 'log|l=s' => \$options{log_file}, - 'pretend|n' => \$options{pretend}, - 'job-servers|j=s@' => \$options{job_servers}, - 'handbrake' => \$options{handbrake}, + 'help|h' => \$options->{help}, + 'log-config|l=s' => \$options->{log_config}, + 'pretend|n' => \$options->{pretend}, + 'job-servers|j=s@' => \$options->{job_servers}, + 'gearman-prefix=s' => \$options->{gearman_prefix}, + 'handbrake' => \$options->{handbrake}, + 'config|c=s' => \$options->{config_file}, ) or pod2usage(-verbose => 0); -pod2usage(-verbose => 1) if ($options{help}); +pod2usage(-verbose => 1) if ($options->{help}); + +# Parse the configuration file (if any), and merge/validate the options +my $config = parse_config($options->{config_file}); +($config, $options) = process_config($config, $options, $default_options); # Setup logging -my $log = Log::Handler->new(); -my $maxLogLevel = ($options{debug} ? 'debug' : ($options{quiet} ? 3 : $options{verbose} + 4)); # default to logging warning+, unless quiet in which case log error+, or debug in which case log everything -my $maxFileLogLevel = ($options{debug} ? 'debug' : 'info'); - -if ( ! $options{silent}) { - $log->add( - screen => { - log_to => 'STDOUT', - minlevel => 'emergency', - maxlevel => $maxLogLevel, - }, - ); -} -if ( $options{log_file}) { - $log->add( - file => { - filename => $options{log_file}, - minlevel => 'emergency', - maxlevel => $maxFileLogLevel, - }, - ); -} +Log::Log4perl->init($options->{log_config}); +my $worker_log = Log::Log4perl->get_logger('HandbrakeCluster::Worker'); +$worker_log->debug("Logging started"); # Setup the worker, and listen for jobs -my $worker = Gearman::Worker->new(job_servers => $options{job_servers}); +my $worker = Gearman::Worker->new(job_servers => $options->{job_servers}); +$worker->prefix($options->{gearman_prefix}) if $options->{gearman_prefix}; $worker->register_function(handbrake_rip => \&handbrake_rip); $worker->work while 1; sub handbrake_rip { my $job = shift; - my $rip_options = thaw($job->arg); + my $rip_options; + if ($options->{php_client}) { + $rip_options = PHP::Serialization::unserialize($job->arg); + } else { + $rip_options = thaw($job->arg); + } my $response = {}; - $log->notice("Beginning new rip to $rip_options->{output_filename}"); + # Setup logging for this job + my $job_log = Log::Log4perl->get_logger('HandbrakeCluster::Worker::Job'); + Log::Log4perl::MDC->put('job_id', $rip_options->{db_job_id}); + $job_log->info("Beginning new rip to $rip_options->{output_filename}"); # Generate a unique filename based on the output filename to prevent clashes from previous runs my $uuid = random_string('cccccc'); @@ -91,7 +88,7 @@ sub handbrake_rip { 'nice', '-n', $rip_options->{nice}, # Construct the handbrake command line - $options{handbrake}, + $options->{handbrake}, get_options($rip_options, 'input_filename', '-i'), get_options($rip_options, 'real_output_filename', '-o'), get_options($rip_options, 'title'), @@ -114,7 +111,7 @@ sub handbrake_rip { $job->set_status(0, 100); # Execute the ripping process - $log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd)); + $job_log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd)); my ($child_in, $child_out, $child_err) = map gensym, 1..3; my $child_pid = open3($child_in, $child_out, $child_err, @handbrake_cmd); # No need to write to the child process @@ -135,11 +132,11 @@ sub handbrake_rip { foreach my $handle (@ready) { my $bytes_read = sysread($handle, my $buf = '', 1024); if ($bytes_read == -1) { - $log->error("Error reading from HandBrake socket: $!"); + $job_log->error("Error reading from HandBrake socket: $!"); $select->remove($child_out); next; } elsif ($bytes_read == 0) { - $log->debug("HandBrake socket closed"); + $job_log->debug("HandBrake socket closed"); $select->remove($handle); next; } @@ -153,8 +150,8 @@ sub handbrake_rip { $child_out_buffer = $lines[1]; if ($line =~ m/Encoding: task \d+ of \d+, (\d+\.\d+) %/) { - my $numerator = $1; - $job->set_status($numerator, 1); + my $numerator = 100 * $1; + $job->set_status($numerator, 100); } } } else { @@ -166,7 +163,7 @@ sub handbrake_rip { $child_err_buffer = $lines[1]; push @{ $response->{log} }, $line; - $log->notice($line); + $job_log->info($line); } } } @@ -182,9 +179,9 @@ sub handbrake_rip { $response->{status} = $? >> 8; $response->{success} = $response->{status} == 0; if ($response->{success}) { - $log->notice("Finished rip to $response->{real_output_filename}"); + $job_log->info("Finished rip to $response->{real_output_filename}"); } else { - $log->warning("Ripping process returned error status: $response->{status}"); + $job_log->warning("Ripping process returned error status: $response->{status}"); } return freeze($response); @@ -217,6 +214,37 @@ sub get_options { } } +# Reads configuration options from a config file, expands the internal references, and returns the expanded form. +sub parse_config { + my $config_file = shift; + + return {} unless defined $config_file; + my $config = YAML::Any::LoadFile($options->{config_file}) or die 'Unable to load configuration file: ' . $!; + + return $config; +} + +sub process_config { + my $config = shift or die; + my $options = shift or die; + my $default_options = shift or die; + + # Update any unset options with values from config file + foreach my $option (keys %$options) { + # Update the value of any option which has not been set by a command line argument + if (!$options->{$option}) { + # Try the config file first, otherwise use the default value + if (defined $config->{options}->{$option}) { + $options->{$option} = $config->{options}->{$option}; + } else { + $options->{$option} = $default_options->{$option}; + } + } + } + + return ($config, $options); +} + __END__; =head1 NAME @@ -225,9 +253,7 @@ __END__; =head1 SYNOPSIS handbrake-cluster-worker.pl -h|--help - handbrake-cluster-worker.pl [[-v|--verbose] | [-d|--debug] | - [-q|--quiet] | [-s|--silent]] - [-l|--log ] + handbrake-cluster-worker.pl [-l|--log-config ] [-n|--pretend] [-j|--job-servers ] [--handbrake ] @@ -246,26 +272,10 @@ job servers to use are configurable by command line arguments. Displays this help information and quits. -=item B<-v|--verbose> +=item B<-l|--log-config Elog4perl config fileE> -Displays verbose logging information. - -=item B<-d|--debug> - -Displays full debugging information, including all output from HandBrake. - -=item B<-q|--quiet> - -Displays only errors. - -=item B<-s|--silent> - -Displays no output whatsoever, useful for scripting. Output will still be -logged to disk if a file is specified. - -=item B<-l|--log Elog fileE> - -Specifies the name of a file to write logging information to. +Specifies the name of a Log4perl configuration file to control logging. +This file is expected to define a logger named HandbrakeCluster::Worker. =item B<-n|--pretend> diff --git a/worker.yml.dist b/worker.yml.dist new file mode 100644 index 0000000..19e9b15 --- /dev/null +++ b/worker.yml.dist @@ -0,0 +1,45 @@ +--- +options: + debug: 1 + log_file: /tmp/handbrake-cluster-client.log + job_servers: + - build0.example.com + - build1.example.com + gearman_prefix: dev + +log_database: + driver: mysql + host: localhost + database: handbrake_cluster + user: handbrake + password: + table: worker_log + columns: + - level + - ctime + - pid + - hostname + - progname + - line + - message + values: + - %level + - %time + - %pid + - %hostname + - %progname + - %line + - %message + message_pattern: + - %L + - %T + - %P + - %H + - %S + - %l + - %m + message_layout: %m + timeformat: %s + +# vim: set ft=yaml ts=2 expandtab : +