commit 0e4f9324aa4b422d795759fe8e0f5c1d112c7a45 Author: Ben Roberts Date: Mon Feb 15 00:16:45 2010 +0000 Initial commits for handbrake-cluster In its current state, the worker thread will process jobs submitted to gearman by one or more clients. All output from HandBrake is captured and logged along with diagnostic output from the worker process. The client process takes commandline arguments to run a single ripping task, and waits for it to complete. diff --git a/handbrake-cluster-client.pl b/handbrake-cluster-client.pl new file mode 100755 index 0000000..97202a0 --- /dev/null +++ b/handbrake-cluster-client.pl @@ -0,0 +1,151 @@ +#!/usr/bin/perl + +use warnings; +use strict; + +use Data::Dumper; +use Gearman::Client; +use Getopt::Long; +use Log::Handler; +use Pod::Usage; +use Storable qw/freeze thaw/; + +# Handle interrupts, and term signals +$SIG{'INT'} = 'INT_handler'; +$SIG{'TERM'} = 'TERM_handler'; + +# Globals +our %options = ( + verbose => 0, + quiet => 0, + log_file => '/tmp/handbrake-cluster-client.log', + silent => 0, + help => 0, + pretend => 0, + job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'], +); +our %rip_options = ( + nice => 15, + input_filename => '/dev/sr0', + output_filename => 'rip-output.mkv', + title => -1, + format => 'mkv', + video_codec => 'x264', + video_width => 720, # DVD resolution + video_height => 576, # DVD resolution + quantizer => 0.61,# x264 quantizer = 20 + deinterlace => 0, # 0 = off, 1 = on, 2 = selective + audio_tracks => 1, + audio_codec => 'ac3', + audio_names => 'English', + subtitle_tracks => 1, +); + +Getopt::Long::Configure( qw(bundling no_getopt_compat) ); +GetOptions( + 'verbose|v+' => \$options{verbose}, + 'debug|d' => \$options{debug}, + 'quiet|q' => \$options{quiet}, + 'silent|s' => \$options{silent}, + 'log|l=s' => \$options{log_file}, + 'help|h' => \$options{help}, + 'pretend|n' => \$options{pretend}, + 'job-servers|j=s@' => \$options{job_servers}, + 'nice|N=i' => \$rip_options{nice}, + 'input|i=s' => \$rip_options{input_filename}, + 'output|o=s' => \$rip_options{output_filename}, + 'title|t=s' => \$rip_options{title}, + 'format|f=s' => \$rip_options{format}, + 'video-encoder|e=s' => \$rip_options{video_codec}, + 'width|w=i' => \$rip_options{video_width}, + 'height|l=i' => \$rip_options{video_height}, + 'quantizer|Q=f' => \$rip_options{quantizer}, + 'deinterlate|D=i' => \$rip_options{deinterlace}, + 'audio-tracks|a=s' => \$rip_options{audio_tracks}, + 'audio-encoder|E=s' => \$rip_options{audio_codec}, + 'audio-name|A=s' => \$rip_options{audio_names}, + 'subtitle-tracks|S=s' => \$rip_options{subtitle_tracks}, +) or pod2usage(-verbose => 0); +pod2usage(-verbose => 1) if ($options{help}); + +# Setup logging +my $log = Log::Handler->new(); +my $maxLogLevel = ($options{debug} ? 'debug' : ($options{quiet} ? 3 : $options{verbose} + 4)); # default to logging warning+, unless quiet in which case log error+, or debug in which case log everything +my $maxFileLogLevel = ($options{debug} ? 'debug' : 'info'); + +if ( ! $options{silent}) { + $log->add( + screen => { + log_to => 'STDOUT', + minlevel => 'emergency', + maxlevel => $maxLogLevel, + }, + ); +} +if ( $options{log_file}) { + $log->add( + file => { + filename => $options{log_file}, + minlevel => 'emergency', + maxlevel => $maxFileLogLevel, + }, + ); +} + +# Setup the distribution client +my $client = Gearman::Client->new; +$client->job_servers($options{job_servers}); + +# Add new ripping task +my $taskset = $client->new_task_set; +$taskset->add_task("rip", freeze(\%rip_options), + { + on_complete => \&on_complete_handler, + on_fail => \&on_fail_handler, + } +); +$taskset->wait; + +sub on_complete_handler { + my $result_ref = shift or die; + + $log->notice("Completed rip to $$result_ref"); +} + +sub on_fail_handler { + $log->error("Rip failed"); +} + + +sub INT_handler { + $log->error("Caught interrupt, exiting."); + exit 0; +} + +sub TERM_handler { + $log->error("Caught SIGTERM, exiting."); + exit 0; +} + + +__END__; +=head1 NAME + + handbrake-cluster-client - Sets up DVD ripping tasks to be distributed + +=head1 SYNOPSIS + + handbrake-cluster-client.pl -h + handbrake-cluster-client.pl [-v [-d]|-q|-s] + [-l logfile] + [-n] + +=head1 DESCRIPTION + + Sets up one or more DVD ripping tasks, and hands them off to gearman for + distribution. Logging and ripping configuration can be controlled with + command line arguments. + +=head1 OPTIONS + +=cut diff --git a/handbrake-cluster-worker.pl b/handbrake-cluster-worker.pl new file mode 100755 index 0000000..ab10991 --- /dev/null +++ b/handbrake-cluster-worker.pl @@ -0,0 +1,156 @@ +#!/usr/bin/perl + +use warnings; +use strict; + +use Data::Dumper; +use Gearman::Worker; +use Getopt::Long; +use IPC::Open2; +use Log::Handler; +use Pod::Usage; +use String::Random qw/random_string/; +use Storable qw/thaw/; + +# Handle interrupts, and term signals +$SIG{'INT'} = 'INT_handler'; +$SIG{'TERM'} = 'TERM_handler'; + +# Globals +our %options = ( + verbose => 0, + quiet => 0, + log_file => '/tmp/handbrake-cluster-worker.log', + silent => 0, + help => 0, + pretend => 0, + job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'], + handbrake => '/usr/bin/HandBrakeCLI', +); + +Getopt::Long::Configure( qw(bundling no_getopt_compat) ); +GetOptions( + 'verbose|v+' => \$options{verbose}, + 'debug|d' => \$options{debug}, + 'quiet|q' => \$options{quiet}, + 'silent|s' => \$options{silent}, + 'log|l=s' => \$options{log_file}, + 'help|h' => \$options{help}, + 'pretend|n' => \$options{pretend}, + 'job-servers|j=s@' => \$options{job_servers}, + 'handbrake' => \$options{handbrake}, +) or pod2usage(-verbose => 0); +pod2usage(-verbose => 1) if ($options{help}); + +# Setup logging +my $log = Log::Handler->new(); +my $maxLogLevel = ($options{debug} ? 'debug' : ($options{quiet} ? 3 : $options{verbose} + 4)); # default to logging warning+, unless quiet in which case log error+, or debug in which case log everything +my $maxFileLogLevel = ($options{debug} ? 'debug' : 'info'); + +if ( ! $options{silent}) { + $log->add( + screen => { + log_to => 'STDOUT', + minlevel => 'emergency', + maxlevel => $maxLogLevel, + }, + ); +} +if ( $options{log_file}) { + $log->add( + file => { + filename => $options{log_file}, + minlevel => 'emergency', + maxlevel => $maxFileLogLevel, + }, + ); +} + +# Setup the worker, and listen for jobs +my $worker = Gearman::Worker->new(job_servers => $options{job_servers}); +$worker->register_function(rip => \&do_rip); +$worker->work while 1; + +sub do_rip { + my $job = shift; + my %rip_options = %{ thaw($job->arg) }; + + $log->notice("Beginning new rip to $rip_options{output_filename}"); + + my $uuid = random_string('cccccc'); + $log->debug("Using $uuid as unique identifier for this job"); + my $rip_filename = $rip_options{output_filename}; + $rip_filename =~ s/\.([^\.]+)$/\.$uuid\.$1/; + + # Generate deinterlace options + my @deinterlace_options; + push @deinterlace_options, '-d' if ($rip_options{deinterlace} == 1); + push @deinterlace_options, '-5' if ($rip_options{deinterlace} == 2); + + # Generate the command line for handbrake + my @handbrake_cmd = ( + 'nice', '-n', $rip_options{nice}, + $options{handbrake}, + '-i', $rip_options{input_filename}, + '-o', $rip_filename, + '-t', $rip_options{title}, + '-f', $rip_options{format}, + '-e', $rip_options{video_codec}, + '-q', $rip_options{quantizer}, + '-w', $rip_options{video_width}, + '-l', $rip_options{video_height}, + @deinterlace_options, + '-a', $rip_options{audio_tracks}, + '-E', $rip_options{audio_codec}, + '-A', $rip_options{audio_names}, + '-s', $rip_options{subtitle_tracks}, + ); + + $log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd)); + + # Execute the ripping process + my ($child_in, $child_out); + my $child_pid = open2($child_out, $child_in, @handbrake_cmd); + # Don't need to write from the child + close($child_in); + + my $line; + while ($line = <$child_out>) { + $log->debug($line); + } + + $log->notice("Finished rip to $rip_filename"); + return $rip_filename; +} + +sub INT_handler { + $log->error("Caught interrupt, exiting."); + exit 0; +} + +sub TERM_handler { + $log->error("Caught SIGTERM, exiting."); + exit 0; +} + + +__END__; +=head1 NAME + + handbrake-cluster-worker - Processes DVD rips farmed out by gearman + +=head1 SYNOPSIS + + handbrake-cluster-worker.pl -h + handbrake-cluster-worker.pl [-v [-d]|-q|-s] + [-l logfile] + [-n] + +=head1 DESCRIPTION + + Processes ripping tasks as requested by a gearman job server. Logging and the + job servers to use are configurable by command line arguments. + +=head1 OPTIONS + +=cut