Latest version with support for webui and database logging

handbrake-cluster-client.pl

@@ -1,12 +1,14 @@
 #!/usr/bin/perl
 
+package HandbrakeCluster::Client;
+
 use warnings;
 use strict;
 
 use Data::Dumper;
 use Gearman::Client;
 use Getopt::Long;
-use Log::Handler;
+use Log::Log4perl;
 use MIME::Lite::TT::HTML;
 use Pod::Usage;
 use Storable qw/freeze thaw/;
@@ -14,22 +16,19 @@ use YAML::Any;
 
 # Globals
 my $default_options = {
-    verbose         => 0,
+    log_config      => 'logging.conf',
-    debug           => 0,
-    quiet           => 0,
-    log_file        => '/tmp/handbrake-cluster-client.log',
-    silent          => 0,
     help            => 0,
     pretend         => 0,
     config_file     => '',
     job_servers     => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
+    gearman_prefix  => '',
     limit           => [],
-    email_target   => '',
+    email_target    => '',
-    email_sender   => '',
+    email_sender    => '',
-    email_subject  => 'Rip completed',
+    email_subject   => 'Rip completed',
-    email_timezone => 'Europe/London',
+    email_timezone  => 'Europe/London',
-    email_html     => 'rip-completed.html',
+    email_html      => 'rip-completed.html',
-    email_text     => 'rip-completed.txt',
+    email_text      => 'rip-completed.txt',
 };
 my $options = { map { $_ => undef } keys %$default_options };
 
@@ -55,14 +54,11 @@ my $rip_options = { map { $_ => undef } keys %$default_rip_options };
 
 Getopt::Long::Configure( qw(bundling no_getopt_compat) );
 GetOptions(
-    'verbose|v+'          => \$options->{verbose}, 
+    'log-config|l=s'      => \$options->{log_config},
-    'debug|d'             => \$options->{debug},
-    'quiet|q'             => \$options->{quiet}, 
-    'silent|s'            => \$options->{silent}, 
-    'log|l=s'             => \$options->{log_file},
     'help|h'              => \$options->{help},
     'pretend|n'           => \$options->{pretend},
     'job-servers|j=s@'    => \$options->{job_servers},
+    'gearman-prefix=s'    => \$options->{gearman_prefix},
     'config|c=s'          => \$options->{config_file},
     'limit|L=s'           => sub { my ($name, $value) = @_; $options->{limit} = [split(/,/, $value)]; },
     'nice|N=i'            => \$rip_options->{nice},
@@ -95,33 +91,14 @@ $jobs{__commandline} = $rip_options if $options->{title};
 die "No rips specified" unless %jobs;
 
 # Setup logging
-my $log = Log::Handler->new();
+Log::Log4perl->init($options->{log_config});
-# default to logging warning+, unless quiet in which case log error+, or debug in which case log everything
+my $client_log = Log::Log4perl->get_logger('HandbrakeCluster::Client');
-my $maxLogLevel     = ($options->{debug} ? 'debug' : ($options->{quiet} ? 3 : $options->{verbose} + 4));
+$client_log->debug("Logging started");
-# Ignore the quiet option when logging to disk
-my $maxFileLogLevel = ($options->{debug} ? 'debug' : 'info');
 
-if ( ! $options->{silent}) {
-    $log->add(
-        screen => {
-            log_to => 'STDOUT',
-            minlevel => 'emergency',
-            maxlevel => $maxLogLevel,
-        },
-    );
-}
-if ( $options->{log_file}) {
-    $log->add(
-        file => {
-            filename => $options->{log_file},
-            minlevel => 'emergency',
-            maxlevel => $maxFileLogLevel,
-        },
-    );
-}
 
 # Setup the distribution client
 my $client = Gearman::Client->new;
+$client->prefix($options->{gearman_prefix}) if $options->{gearman_prefix};
 $client->job_servers($options->{job_servers});
 
 # Add new ripping task for each job to run
@@ -131,7 +108,13 @@ foreach my $job_id (keys %jobs) {
     my $job = $jobs{$job_id};
 
     # Check that the job hasn't been restricted by the limit option
-    next if (!$options->{limit} || !$options->{job_permitted}->{$job_id});
+    if (@{ $options->{limit} } && !$options->{job_permitted}->{$job_id}) {
+        $client_log->info("Skipped job $job_id due to limit restrictions");
+        next;
+    }
+
+    # Create a new database record for this job, and grab the new id
+    $job->{db_job_id} = 43;
 
     $progress{$job_id} = 0;
 
@@ -148,14 +131,14 @@ foreach my $job_id (keys %jobs) {
             on_complete => \&on_complete_handler,
             on_retry    => sub {
                 my $attempt = shift or die;
-                $log->warning("Retrying '$job_id' rip");
+                $client_log->warning("Retrying '$job_id' rip");
             },
             on_fail     => sub {
-                $log->warning("Rip '$job_id' failed");
+                $client_log->warning("Rip '$job_id' failed");
             },
         }
     );
-    $log->info("Enqueued '$job_id' rip");
+    $client_log->info("Enqueued '$job_id' rip");
 }
 $taskset->wait;
 
@@ -188,7 +171,7 @@ sub on_complete_handler {
         $email->send;
     }
 
-    $log->notice("Completed rip to $response->{real_output_filename}");
+    $client_log->info("Completed rip to $response->{real_output_filename}");
 }
 
 sub display_progress {

handbrake-cluster-worker.pl

@@ -1,5 +1,7 @@
 #!/usr/bin/perl
 
+package HandbrakeCluster::Worker;
+
 use warnings;
 use strict;
 
@@ -9,75 +11,70 @@ use Getopt::Long;
 use IO::Select;
 use IO::Socket;
 use IPC::Open3;
-use Log::Handler;
+use Log::Log4perl;
+use PHP::Serialization;
 use Pod::Usage;
 use String::Random qw/random_string/;
 use Storable qw/freeze thaw/;
 use Switch;
 use Symbol qw/gensym/;
+use YAML::Any;
 
 # Globals
-our %options = (
+our $default_options = {
-    verbose     => 0,
+    log_config     => 'logging.conf',
-    quiet       => 0,
+    help           => 0,
-    log_file    => '/tmp/handbrake-cluster-worker.log',
+    pretend        => 0,
-    silent      => 0,
+    job_servers    => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
-    help        => 0,
+    gearman_prefix => '',
-    pretend     => 0,
+    handbrake      => '/usr/bin/HandBrakeCLI',
-    job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
+    config_file    => '',
-    handbrake   => '/usr/bin/HandBrakeCLI',
+    php_client     => 0,
-);
+};
+my $options = { map { $_ => undef } keys %$default_options };
 
 Getopt::Long::Configure( qw(bundling no_getopt_compat) );
 GetOptions(
-    'help|h'           => \$options{help},
+    'help|h'           => \$options->{help},
-    'verbose|v'        => \$options{verbose}, 
+    'log-config|l=s'   => \$options->{log_config},
-    'debug|d'          => \$options{debug},
+    'pretend|n'        => \$options->{pretend},
-    'quiet|q'          => \$options{quiet}, 
+    'job-servers|j=s@' => \$options->{job_servers},
-    'silent|s'         => \$options{silent}, 
+    'gearman-prefix=s' => \$options->{gearman_prefix},
-    'log|l=s'          => \$options{log_file},
+    'handbrake'        => \$options->{handbrake},
-    'pretend|n'        => \$options{pretend},
+    'config|c=s'       => \$options->{config_file},
-    'job-servers|j=s@' => \$options{job_servers},
-    'handbrake'        => \$options{handbrake},
 ) or pod2usage(-verbose => 0);
-pod2usage(-verbose => 1) if ($options{help});
+pod2usage(-verbose => 1) if ($options->{help});
+
+# Parse the configuration file (if any), and merge/validate the options
+my $config = parse_config($options->{config_file});
+($config, $options) = process_config($config, $options, $default_options);
 
 # Setup logging
-my $log = Log::Handler->new();
+Log::Log4perl->init($options->{log_config});
-my $maxLogLevel     = ($options{debug} ? 'debug' : ($options{quiet} ? 3 : $options{verbose} + 4)); # default to logging warning+, unless quiet in which case log error+, or debug in which case log everything
+my $worker_log = Log::Log4perl->get_logger('HandbrakeCluster::Worker');
-my $maxFileLogLevel = ($options{debug} ? 'debug' : 'info');
+$worker_log->debug("Logging started");
-
-if ( ! $options{silent}) {
-    $log->add(
-        screen => {
-            log_to => 'STDOUT',
-            minlevel => 'emergency',
-            maxlevel => $maxLogLevel,
-        },
-    );
-}
-if ( $options{log_file}) {
-    $log->add(
-        file => {
-            filename => $options{log_file},
-            minlevel => 'emergency',
-            maxlevel => $maxFileLogLevel,
-        },
-    );
-}
 
 # Setup the worker, and listen for jobs
-my $worker = Gearman::Worker->new(job_servers => $options{job_servers});
+my $worker = Gearman::Worker->new(job_servers => $options->{job_servers});
+$worker->prefix($options->{gearman_prefix}) if $options->{gearman_prefix};
 $worker->register_function(handbrake_rip => \&handbrake_rip);
 $worker->work while 1;
 
 sub handbrake_rip {
     my $job = shift;
-    my $rip_options = thaw($job->arg);
+    my $rip_options;
+    if ($options->{php_client}) {
+        $rip_options = PHP::Serialization::unserialize($job->arg);
+    } else {
+        $rip_options = thaw($job->arg);
+    }
 
     my $response = {};
 
-    $log->notice("Beginning new rip to $rip_options->{output_filename}");
+    # Setup logging for this job
+    my $job_log = Log::Log4perl->get_logger('HandbrakeCluster::Worker::Job');
+    Log::Log4perl::MDC->put('job_id', $rip_options->{db_job_id});
+    $job_log->info("Beginning new rip to $rip_options->{output_filename}");
 
     # Generate a unique filename based on the output filename to prevent clashes from previous runs
     my $uuid = random_string('cccccc');
@@ -91,7 +88,7 @@ sub handbrake_rip {
         'nice', '-n', $rip_options->{nice},
 
         # Construct the handbrake command line
-        $options{handbrake},
+        $options->{handbrake},
         get_options($rip_options, 'input_filename', '-i'),
         get_options($rip_options, 'real_output_filename', '-o'),
         get_options($rip_options, 'title'),
@@ -114,7 +111,7 @@ sub handbrake_rip {
     $job->set_status(0, 100);
 
     # Execute the ripping process
-    $log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd));
+    $job_log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd));
     my ($child_in, $child_out, $child_err) = map gensym, 1..3;
     my $child_pid = open3($child_in, $child_out, $child_err, @handbrake_cmd);
     # No need to write to the child process
@@ -135,11 +132,11 @@ sub handbrake_rip {
         foreach my $handle (@ready) {
             my $bytes_read = sysread($handle, my $buf = '', 1024);
             if ($bytes_read == -1) {
-                $log->error("Error reading from HandBrake socket: $!");
+                $job_log->error("Error reading from HandBrake socket: $!");
                 $select->remove($child_out);
                 next;
             } elsif ($bytes_read == 0) {
-                $log->debug("HandBrake socket closed");
+                $job_log->debug("HandBrake socket closed");
                 $select->remove($handle);
                 next;
             }
@@ -153,8 +150,8 @@ sub handbrake_rip {
                     $child_out_buffer = $lines[1];
 
                     if ($line =~ m/Encoding: task \d+ of \d+, (\d+\.\d+) %/) {
-                        my $numerator = $1;
+                        my $numerator = 100 * $1;
-                        $job->set_status($numerator, 1);
+                        $job->set_status($numerator, 100);
                     }
                 }
             } else {
@@ -166,7 +163,7 @@ sub handbrake_rip {
                     $child_err_buffer = $lines[1];
 
                     push @{ $response->{log} }, $line;
-                    $log->notice($line);
+                    $job_log->info($line);
                 }
             }
         }
@@ -182,9 +179,9 @@ sub handbrake_rip {
     $response->{status} = $? >> 8;
     $response->{success} = $response->{status} == 0;
     if ($response->{success}) {
-        $log->notice("Finished rip to $response->{real_output_filename}");
+        $job_log->info("Finished rip to $response->{real_output_filename}");
     } else {
-        $log->warning("Ripping process returned error status: $response->{status}");
+        $job_log->warning("Ripping process returned error status: $response->{status}");
     }
 
     return freeze($response);
@@ -217,6 +214,37 @@ sub get_options {
     }
 }
 
+# Reads configuration options from a config file, expands the internal references, and returns the expanded form.
+sub parse_config {
+    my $config_file = shift;
+
+    return {} unless defined $config_file;
+    my $config = YAML::Any::LoadFile($options->{config_file}) or die 'Unable to load configuration file: ' . $!;
+
+    return $config;
+}
+
+sub process_config {
+    my $config = shift or die;
+    my $options = shift or die;
+    my $default_options = shift or die;
+
+    # Update any unset options with values from config file
+    foreach my $option (keys %$options) {
+        # Update the value of any option which has not been set by a command line argument
+        if (!$options->{$option}) {
+            # Try the config file first, otherwise use the default value
+            if (defined $config->{options}->{$option}) {
+                $options->{$option} = $config->{options}->{$option};
+            } else {
+                $options->{$option} = $default_options->{$option};
+            }
+        }
+    }
+
+    return ($config, $options);
+}
+
 __END__;
 =head1 NAME
 
@@ -225,9 +253,7 @@ __END__;
 =head1 SYNOPSIS
 
  handbrake-cluster-worker.pl -h|--help
- handbrake-cluster-worker.pl [[-v|--verbose] | [-d|--debug] |
+ handbrake-cluster-worker.pl [-l|--log-config <log4perl config file>]
-                              [-q|--quiet] | [-s|--silent]]
-                             [-l|--log <log file>]
                              [-n|--pretend]
                              [-j|--job-servers <server list>]
                              [--handbrake <handbrake executable>]
@@ -246,26 +272,10 @@ job servers to use are configurable by command line arguments.
 
 Displays this help information and quits.
 
-=item B<-v|--verbose>
+=item B<-l|--log-config E<lt>log4perl config fileE<gt>>
 
-Displays verbose logging information.
+Specifies the name of a Log4perl configuration file to control logging.
-
+This file is expected to define a logger named HandbrakeCluster::Worker.
-=item B<-d|--debug>
-
-Displays full debugging information, including all output from HandBrake.
-
-=item B<-q|--quiet>
-
-Displays only errors.
-
-=item B<-s|--silent>
-
-Displays no output whatsoever, useful for scripting. Output will still be
-logged to disk if a file is specified.
-
-=item B<-l|--log E<lt>log fileE<gt>>
-
-Specifies the name of a file to write logging information to.
 
 =item B<-n|--pretend>
 

worker.yml.dist

@@ -0,0 +1,45 @@
+---
+options:
+    debug:          1
+    log_file:       /tmp/handbrake-cluster-client.log
+    job_servers:
+      - build0.example.com
+      - build1.example.com
+    gearman_prefix: dev
+
+log_database:
+  driver:           mysql
+  host:             localhost
+  database:         handbrake_cluster
+  user:             handbrake
+  password:         
+  table:            worker_log
+  columns:
+    - level
+    - ctime
+    - pid
+    - hostname
+    - progname
+    - line
+    - message
+  values:
+    - %level
+    - %time
+    - %pid
+    - %hostname
+    - %progname
+    - %line
+    - %message
+  message_pattern:
+    - %L
+    - %T
+    - %P
+    - %H
+    - %S
+    - %l
+    - %m
+  message_layout:   %m
+  timeformat:       %s
+  
+# vim: set ft=yaml ts=2 expandtab :
+