diff --git a/config.yml b/config.yml index 77c4f03..9103998 100644 --- a/config.yml +++ b/config.yml @@ -5,11 +5,11 @@ options: job_servers: - build0.example.com - build1.example.com + nice: 15 report_email: me@example.com presets: tvseries: - nice: 15 format: mkv video_codec: x264 video_width: 720 diff --git a/handbrake-cluster-client.pl b/handbrake-cluster-client.pl index ff2187c..853ee5c 100755 --- a/handbrake-cluster-client.pl +++ b/handbrake-cluster-client.pl @@ -12,12 +12,8 @@ use Pod::Usage; use Storable qw/freeze thaw/; use YAML::Any; -# Handle interrupts, and term signals -$SIG{'INT'} = 'INT_handler'; -$SIG{'TERM'} = 'TERM_handler'; - # Globals -our %default_options = ( +my $default_options = { verbose => 0, debug => 0, quiet => 0, @@ -25,13 +21,18 @@ our %default_options = ( silent => 0, help => 0, pretend => 0, - job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'], - report_email => '', config_file => '', -); -our %options = map { $_ => undef } keys %default_options; + job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'], + email_target => '', + email_sender => '', + email_subject => 'Rip completed', + email_timezone => 'Europe/London', + email_html => 'rip-completed.html', + email_text => 'rip-completed.txt', +}; +my $options = { map { $_ => undef } keys %$default_options }; -our %rip_options = ( +my $rip_options = { nice => 15, input_filename => '/dev/sr0', output_filename => 'rip-output.mkv', @@ -46,56 +47,57 @@ our %rip_options = ( audio_codec => 'ac3', audio_names => 'English', subtitle_tracks => 1, -); +}; Getopt::Long::Configure( qw(bundling no_getopt_compat) ); GetOptions( - 'verbose|v+' => \$options{verbose}, - 'debug|d' => \$options{debug}, - 'quiet|q' => \$options{quiet}, - 'silent|s' => \$options{silent}, - 'log|l=s' => \$options{log_file}, - 'help|h' => \$options{help}, - 'pretend|n' => \$options{pretend}, - 'job-servers|j=s@' => \$options{job_servers}, - 'report|r=s' => \$options{report_email}, - 'config|c=s' => \$options{config_file}, - 'nice|N=i' => \$rip_options{nice}, - 'input|i=s' => \$rip_options{input_filename}, - 'output|o=s' => \$rip_options{output_filename}, - 'title|t=s' => \$rip_options{title}, - 'format|f=s' => \$rip_options{format}, - 'video-encoder|e=s' => \$rip_options{video_codec}, - 'width|w=i' => \$rip_options{video_width}, - 'height|l=i' => \$rip_options{video_height}, - 'quantizer|Q=f' => \$rip_options{quantizer}, - 'deinterlate|D=i' => \$rip_options{deinterlace}, - 'audio-tracks|a=s' => \$rip_options{audio_tracks}, - 'audio-encoder|E=s' => \$rip_options{audio_codec}, - 'audio-name|A=s' => \$rip_options{audio_names}, - 'subtitle-tracks|S=s' => \$rip_options{subtitle_tracks}, + 'verbose|v+' => \$options->{verbose}, + 'debug|d' => \$options->{debug}, + 'quiet|q' => \$options->{quiet}, + 'silent|s' => \$options->{silent}, + 'log|l=s' => \$options->{log_file}, + 'help|h' => \$options->{help}, + 'pretend|n' => \$options->{pretend}, + 'job-servers|j=s@' => \$options->{job_servers}, + 'config|c=s' => \$options->{config_file}, + 'nice|N=i' => \$rip_options->{nice}, + 'input|i=s' => \$rip_options->{input_filename}, + 'output|o=s' => \$rip_options->{output_filename}, + 'title|t=s' => \$rip_options->{title}, + 'format|f=s' => \$rip_options->{format}, + 'video-encoder|e=s' => \$rip_options->{video_codec}, + 'width|w=i' => \$rip_options->{video_width}, + 'height|l=i' => \$rip_options->{video_height}, + 'quantizer|Q=f' => \$rip_options->{quantizer}, + 'deinterlate|D=i' => \$rip_options->{deinterlace}, + 'audio-tracks|a=s' => \$rip_options->{audio_tracks}, + 'audio-encoder|E=s' => \$rip_options->{audio_codec}, + 'audio-name|A=s' => \$rip_options->{audio_names}, + 'subtitle-tracks|S=s' => \$rip_options->{subtitle_tracks}, + 'email-target=s' => \$options->{email_target}, + 'email-sender=s' => \$options->{email_sender}, + 'email-subject=s' => \$options->{email_subject}, ) or pod2usage(-verbose => 0); -pod2usage(-verbose => 1) if ($options{help}); +pod2usage(-verbose => 1) if ($options->{help}); +# Parse the configuration file (if any), and merge/validate the options +my $config = parse_config($options->{config_file}); +($config, $options) = process_config($config, $options, $default_options); + +# Create a list of jobs from the configuration file, and command line options my @jobs; -my $config; -# Read a configuration file, if provided -if ($options{config_file}) { - $config = parse_config($options{config_file}); - push @jobs, @{ $config->{jobs} }; -} - -# A list of jobs to run, formed from command line options or config file -if ($rip_options{title}) { - push @jobs, \%rip_options; -} +push @jobs, @{ $config->{jobs} } if defined $config->{jobs}; +push @jobs, $rip_options if $options->{title}; +die "No rips specified" unless @jobs; # Setup logging my $log = Log::Handler->new(); -my $maxLogLevel = ($options{debug} ? 'debug' : ($options{quiet} ? 3 : $options{verbose} + 4)); # default to logging warning+, unless quiet in which case log error+, or debug in which case log everything -my $maxFileLogLevel = ($options{debug} ? 'debug' : 'info'); +# default to logging warning+, unless quiet in which case log error+, or debug in which case log everything +my $maxLogLevel = ($options->{debug} ? 'debug' : ($options->{quiet} ? 3 : $options->{verbose} + 4)); +# Ignore the quiet option when logging to disk +my $maxFileLogLevel = ($options->{debug} ? 'debug' : 'info'); -if ( ! $options{silent}) { +if ( ! $options->{silent}) { $log->add( screen => { log_to => 'STDOUT', @@ -104,10 +106,10 @@ if ( ! $options{silent}) { }, ); } -if ( $options{log_file}) { +if ( $options->{log_file}) { $log->add( file => { - filename => $options{log_file}, + filename => $options->{log_file}, minlevel => 'emergency', maxlevel => $maxFileLogLevel, }, @@ -116,12 +118,12 @@ if ( $options{log_file}) { # Setup the distribution client my $client = Gearman::Client->new; -$client->job_servers($options{job_servers}); +$client->job_servers($options->{job_servers}); # Add new ripping task for each job to run my $taskset = $client->new_task_set; foreach my $job (@jobs) { - $taskset->add_task("rip", freeze($job), + $taskset->add_task("handbrake_rip", freeze($job), { on_complete => \&on_complete_handler, on_fail => \&on_fail_handler, @@ -132,57 +134,55 @@ $taskset->wait; sub on_complete_handler { my $result_ref = shift or die; + my $response = thaw($$result_ref); - return on_fail_handler() unless defined $$result_ref; - - if ($options{report_email}) { + if ($options->{email_target}) { my $email = MIME::Lite::TT::HTML->new( - From => $options{report_email},, - To => $options{report_email}, - Subject => 'Rip completed', - TimeZone => 'Europe/London', + From => $options->{email_sender}, + To => $options->{email_target}, + Subject => $options->{email_subject}, + TimeZone => $options->{email_timezone}, Encoding => 'quoted-printable', Template => { - html => 'rip-completed.html', - text => 'rip-completed.txt', + html => $options->{email_html}, + text => $options->{email_text}, }, Charset => 'utf8', - TmplOptions => {}, - TmplParams => {}, + TmplOptions => { + INCLUDE_PATH => '.', + }, + TmplParams => { + success => $response->{success}, + filename => $response->{output_filename}, + real_filename => $response->{real_output_filename}, + log => join '\n', @{ $response->{log} }, + }, ); $email->send; } - $log->notice("Completed rip to $$result_ref"); + $log->notice("Completed rip to $response->{real_output_filename}"); } sub on_fail_handler { - $log->error("Rip failed"); + $log->error("Failed to distribute job"); } +# Reads configuration options from a config file, expands the internal references, and returns the expanded form. sub parse_config { - my $config_file = shift or die; - my $config = YAML::Any::LoadFile($options{config_file}) or die 'Unable to load configuration file: ' . $!; + my $config_file = shift; - # Update any unset options with values from config file - foreach my $option (keys %options) { - # Update the value of any option which has not been set by a command line argument - if (!$options{$option}) { - # Try the config file first, otherwise use the default value - if (defined $config->{options}->{$option}) { - $options{$option} = $config->{options}->{$option}; - } else { - $options{$option} = $default_options{$option}; - } - } - } + return {} unless defined $config_file; + my $config = YAML::Any::LoadFile($options->{config_file}) or die 'Unable to load configuration file: ' . $!; # Iterate through each job, and inject any preset variables that haven't been redefined by the job - foreach my $job (@{ $config->{jobs} }) { - if ($job->{presets}) { - foreach my $preset_name (@{ $job->{presets} }) { - foreach my $preset_key (keys %{$config->{presets}->{$preset_name}}) { - $job->{$preset_key} = $config->{presets}->{$preset_name}->{$preset_key} unless $job->{$preset_key}; + if (defined $config->{jobs}) { + foreach my $job (@{ $config->{jobs} }) { + if ($job->{presets}) { + foreach my $preset_name (@{ $job->{presets} }) { + foreach my $preset_key (keys %{$config->{presets}->{$preset_name}}) { + $job->{$preset_key} = $config->{presets}->{$preset_name}->{$preset_key} unless $job->{$preset_key}; + } } } } @@ -191,15 +191,30 @@ sub parse_config { return $config; } +sub process_config { + my $config = shift or die; + my $options = shift or die; + my $default_options = shift or die; -sub INT_handler { - $log->error("Caught interrupt, exiting."); - exit 0; -} + # Update any unset options with values from config file + foreach my $option (keys %$options) { + # Update the value of any option which has not been set by a command line argument + if (!$options->{$option}) { + # Try the config file first, otherwise use the default value + if (defined $config->{options}->{$option}) { + $options->{$option} = $config->{options}->{$option}; + } else { + $options->{$option} = $default_options->{$option}; + } + } + } -sub TERM_handler { - $log->error("Caught SIGTERM, exiting."); - exit 0; + # Validate the email options + if ($options->{email_target}) { + $options->{email_sender} = $options->{email_target} unless $options->{email_sender}; + } + + return ($config, $options); } @@ -210,17 +225,224 @@ __END__; =head1 SYNOPSIS - handbrake-cluster-client.pl -h - handbrake-cluster-client.pl [-v [-d]|-q|-s] - [-l logfile] - [-n] + handbrake-cluster-client.pl -h|--help + handbrake-cluster-client.pl [[-v|--verbose] | [-d|--debug] | + [-q|--quiet]] | [-s|--silent]] + [-l|--log ] + [-n|--pretend] + [-c|--config ] + [-N|--nice ] + [-i|--input ] + [-o|--output ] + [-f|--format ] + [-t|--title ] + [-e|--video-encoder <ffmpeg|theora|x264>] + [-w|--width <width>] + [-l|--height <height>] + [-Q|--quantizer <quality>] + [-D|--deinterlace <0|1|2>] + [-a|--audio-tracks <track list>] + [-E|--audio-encoder + <faac/lame/vorbis/ac3/dts>[,...]] + [-A|--audio-name <audio name list>] + [-s|--subtitle-tracks <subtitle track list>] + [--email-target <email address>] + [--email-sender <email address>] + [--email-subject <subject line>] =head1 DESCRIPTION - Sets up one or more DVD ripping tasks, and hands them off to gearman for - distribution. Logging and ripping configuration can be controlled with - command line arguments. +Sets up one or more DVD ripping tasks, and hands them off to gearman for +distribution. Logging and ripping configuration can be controlled with +command line arguments. =head1 OPTIONS +=head2 Administrative Options + +=over 4 + +=item B<-h|--help> + +Displays this help information and quits. + +=item B<-v|--verbose> + +Displays verbose logging information. + +=item B<-d|--debug> + +Displays full debugging information, including all output from HandBrake. + +=item B<-q|--quiet> + +Displays only errors. + +=item B<-s|--silent> + +Displays no output whatsoever, useful for scripting. Output will still be +logged to disk if a file is specified. + +=item B<-l|--log E<lt>log fileE<gt>> + +Specifies the name of a file to write logging information to. + +=item B<-n|--pretend> + +Only shows what action would be taken, no rips are actually performed. + +=item B<-j|--job-servers E<lt>server listE<gt>> + +Specifies a comma separated list of alternate servers to use for job +distribution. + +=item B<-c|--config E<lt>config fileE<gt>> + +Specifies the name of a configuration file which contains additional options, +and a list of ripping jobs. + +=item B<-n|--nice E<lt>nice levelE<gt>> + +Specifies the priority to give the ripping process, using nice. +Defaults to 15. + +=back + +=head2 Ripping Options + +=over 4 + +=item B<-i|--input E<lt>input filenameE<gt>> + +Specifies the DVD drive device, or path to a VIDEO_TS folder to use as input +for the rip. + +=item B<-o|--output E<lt>output filenameE<gt>> + +Specifies the name of the file the rip will be saved to. This is only used +as a template; the actual rip will be saved to a filename with a unique +component at the end to avoid accidental overwrites. + +=item B<-f|--format E<lt>mkv|mp4E<gt>> + +Selects the container used for the rip. Defaults to mkv. + +=item B<-t|--title E<lt>titleE<gt>> + +Specifies the DVD title to rip. By default, the longest title will be used. + +=item B<-e|--video-encoder E<lt>ffmpeg|theora|x264E<gt>> + +Specifies the video encoder to use for the rip. Defaults to x264. + +=item B<-w|--width E<lt>widthE<gt>> + +Specifies the width of the output rip in pixels. + +=item B<-l|--height E<lt>heightE<gt>> + +Specifies the height of the output rip in pixels. + +=item B<-Q|--quantizer E<lt>qualityE<gt>> + +Specifies the x264 quantizer property, in the form of a real between 0 and 1. +Defaults to 0.61, aka "20". + +=item B<D|--deinterlace E<lt>0|1|2E<gt>> + +Specifies the type of deinterlacing to use. Use 0 to disable completely. +Use 1 for full deinterlacing, or 2 for selective deinterlacing (recommended). +Defaults to 0. + +=item B<-a|--audio-tracks E<lt>audio track listE<gt>> + +Specifies a comma separated list of audio tracks to include in the rip. +Defaults to 1. + +=item B<-E|--audio-encoder E<lt>faac/lame/vorbis/ac3/dts>E<gt>> + +Specifies the audio encoder to use for the rip. Defaults to ac3 pass-through. + +=item B<-A|--audio-name E<lt>audio track namesE<gt>> + +Specifies a comma separated list of names for the selected audio tracks. +Defaults to 'English'. + +=back + +=head2 Reporting Options + +=over 4 + +=item B<--email-target E<lt>email addressE<gt>> + +Specifies an email address to send a completion report to after each rip +finishes. + +=item B<--email-sender E<lt>email addressE<gt>> + +Specifies an email address to use as the sender of the rip completion email +reports. Defaults to the same as the target address. + +=item B<--email-subject E<lt>subject lineE<gt>> + +Overrides the subject line to use in the rip completion email reports. + +=back + +=head1 CONFIGURATION + +This script accepts a configuration file in YAML format to specify multiple +ripping tasks, or provide values for commonly used options. + +Command line arguments take precedence to options specified in config files. + +This file is split into three sections: + +=over 4 + +=item B<options> + +Values for command-line options from the I<Administrative Options> section +above can be specified here, using variables with the same name as the long +options (with hyphens replaced with underscores). + + options: + job_servers: build0.example.com + log_file: /tmp/handbrake.log + +=item B<presets> + +Presets allow the grouping of common job configuration options, for example +the input filename when ripping multiple episodes from a single DVD image. +Any option from the I<Ripping Options> section can be specified in a preset, +and options specified in a job take precedence in case of a clash. + + preset: + disk1: + input_filename: /tmp/disk1/VIDEO_TS + tvseries: + format: mkv + video_codec: x264 + +=item B<jobs> + +Specifies a list of ripping tasks to perform. Variables from the +I<Ripping Options> section above can be used here. + +Additionally, you can include all of the options from one or more presets. + + jobs: + - presets: + - disk1 + - tvseries + output_filename: episode1.mkv + +=back + +=head2 EXAMPLE CONFIGURATION + +An example configuration file is included in the distribution as C<config.yml>. +Refer to this when writing your own files. + =cut diff --git a/handbrake-cluster-worker.pl b/handbrake-cluster-worker.pl index 17f7d43..cfe59d6 100755 --- a/handbrake-cluster-worker.pl +++ b/handbrake-cluster-worker.pl @@ -10,13 +10,9 @@ use IPC::Open2; use Log::Handler; use Pod::Usage; use String::Random qw/random_string/; -use Storable qw/thaw/; +use Storable qw/freeze thaw/; use Switch; -# Handle interrupts, and term signals -$SIG{'INT'} = 'INT_handler'; -$SIG{'TERM'} = 'TERM_handler'; - # Globals our %options = ( verbose => 0, @@ -31,12 +27,12 @@ our %options = ( Getopt::Long::Configure( qw(bundling no_getopt_compat) ); GetOptions( - 'verbose|v+' => \$options{verbose}, + 'help|h' => \$options{help}, + 'verbose|v' => \$options{verbose}, 'debug|d' => \$options{debug}, 'quiet|q' => \$options{quiet}, 'silent|s' => \$options{silent}, 'log|l=s' => \$options{log_file}, - 'help|h' => \$options{help}, 'pretend|n' => \$options{pretend}, 'job-servers|j=s@' => \$options{job_servers}, 'handbrake' => \$options{handbrake}, @@ -69,19 +65,22 @@ if ( $options{log_file}) { # Setup the worker, and listen for jobs my $worker = Gearman::Worker->new(job_servers => $options{job_servers}); -$worker->register_function(rip => \&do_rip); +$worker->register_function(handbrake_rip => \&handbrake_rip); $worker->work while 1; -sub do_rip { +sub handbrake_rip { my $job = shift; my %rip_options = %{ thaw($job->arg) }; + my %response; + $log->notice("Beginning new rip to $rip_options{output_filename}"); + # Generate a unique filename based on the output filename to prevent clashes from previous runs my $uuid = random_string('cccccc'); - $log->debug("Using $uuid as unique identifier for this job"); - $rip_options{unique_output_filename} = $rip_options{output_filename}; - $rip_options{unique_output_filename} =~ s/\.([^\.]+)$/\.$uuid\.$1/; + $rip_options{real_output_filename} = $rip_options{output_filename}; + $rip_options{real_output_filename} =~ s/\.([^\.]+)$/\.$uuid\.$1/; + $response{real_output_filename} = $rip_options{real_output_filename}; # Generate the command line for handbrake my @handbrake_cmd = ( @@ -91,7 +90,7 @@ sub do_rip { # Construct the handbrake command line $options{handbrake}, get_options(\%rip_options, 'input_filename', '-i'), - get_options(\%rip_options, 'unique_output_filename', '-o'), + get_options(\%rip_options, 'real_output_filename', '-o'), get_options(\%rip_options, 'title'), get_options(\%rip_options, 'format', '-f'), get_options(\%rip_options, 'video_codec', '-e'), @@ -105,31 +104,36 @@ sub do_rip { get_options(\%rip_options, 'subtitle_tracks', '-s'), ); - $log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd)); + # Return a copy of the command used to rip the title + $response{handbrake_cmd} = @handbrake_cmd; # Execute the ripping process + $log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd)); my ($child_in, $child_out); my $child_pid = open2($child_out, $child_in, @handbrake_cmd); - # Don't need to write from the child + # No need to write to the child process close($child_in); + # Log the output from handbrake, and return it back to the client + $response{log} = (); my $line; while ($line = <$child_out>) { + push @{ $response{log} }, $line; $log->debug($line); } close($child_out); # If the rip process failed, report an error status here waitpid($child_pid, 0); - my $child_exit_status = $? >> 8; - - if ($child_exit_status) { - $log->warning("Ripping process returned error status: $child_exit_status"); - return undef; + $response{status} = $? >> 8; + $response{success} = $response{status} == 0; + if ($response{success}) { + $log->notice("Finished rip to $rip_options{real_output_filename}"); + } else { + $log->warning("Ripping process returned error status: $response{status}"); } - $log->notice("Finished rip to $rip_options{unique_output_filename}"); - return $rip_options{unique_output_filename}; + return freeze(\%response); } sub get_options { @@ -155,17 +159,6 @@ sub get_options { } } -sub INT_handler { - $log->error("Caught interrupt, exiting."); - exit 0; -} - -sub TERM_handler { - $log->error("Caught SIGTERM, exiting."); - exit 0; -} - - __END__; =head1 NAME @@ -173,16 +166,61 @@ __END__; =head1 SYNOPSIS - handbrake-cluster-worker.pl -h - handbrake-cluster-worker.pl [-v [-d]|-q|-s] - [-l logfile] - [-n] + handbrake-cluster-worker.pl -h|--help + handbrake-cluster-worker.pl [[-v|--verbose] | [-d|--debug] | + [-q|--quiet] | [-s|--silent]] + [-l|--log <log file>] + [-n|--pretend] + [-j|--job-servers <server list>] + [--handbrake <handbrake executable>] + =head1 DESCRIPTION - Processes ripping tasks as requested by a gearman job server. Logging and the - job servers to use are configurable by command line arguments. +Processes ripping tasks as requested by a gearman job server. Logging and the +job servers to use are configurable by command line arguments. =head1 OPTIONS +=over 4 + +=item B<-h|--help> + +Displays this help information and quits. + +=item B<-v|--verbose> + +Displays verbose logging information. + +=item B<-d|--debug> + +Displays full debugging information, including all output from HandBrake. + +=item B<-q|--quiet> + +Displays only errors. + +=item B<-s|--silent> + +Displays no output whatsoever, useful for scripting. Output will still be +logged to disk if a file is specified. + +=item B<-l|--log E<lt>log fileE<gt>> + +Specifies the name of a file to write logging information to. + +=item B<-n|--pretend> + +Only shows what action would be taken, no rips are actually performed. + +=item B<-j|--job-servers E<lt>server listE<gt>> + +Specifies a comma separated list of alternate servers to use for job +distribution. + +=item B<--handbrake E<lt>handbrake executableE<gt>> + +Specifies an alternate HandBrake executable to use. Default is +C</usr/bin/HandBrakeCLI>. + =cut diff --git a/rip-completed.html b/rip-completed.html index 0b2757b..2dcab13 100644 --- a/rip-completed.html +++ b/rip-completed.html @@ -1 +1,9 @@ -A rip has completed successfully +[% IF success %] +A HandBrake rip to [% real_filename %] has completed successfully. +[% ELSE %] +A HandBrake rip to [% real_filename %] has failed. +[% END %] + +A log of the activity is included below. + +[% log %] diff --git a/rip-completed.txt b/rip-completed.txt index 0b2757b..2dcab13 100644 --- a/rip-completed.txt +++ b/rip-completed.txt @@ -1 +1,9 @@ -A rip has completed successfully +[% IF success %] +A HandBrake rip to [% real_filename %] has completed successfully. +[% ELSE %] +A HandBrake rip to [% real_filename %] has failed. +[% END %] + +A log of the activity is included below. + +[% log %]