ben/handbrake-cluster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Massive update tidying codebase for release

Browse Source 
Added POD to client and worker
Updated HTMl and Text email templates
This commit is contained in:
Ben Roberts
2010-02-19 22:07:45 +00:00
parent 35b5968b33
commit f71987cbb6
5 changed files with 418 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config.yml
View File

@@ -5,11 +5,11 @@ options:
job_servers:
- build0.example.com
- build1.example.com
nice: 15
report_email: me@example.com
presets:
tvseries:
nice: 15
format: mkv
video_codec: x264
video_width: 720

404
handbrake-cluster-client.pl
View File

@@ -12,12 +12,8 @@ use Pod::Usage;
use Storable qw/freeze thaw/;
use YAML::Any;
# Handle interrupts, and term signals
$SIG{'INT'} = 'INT_handler';
$SIG{'TERM'} = 'TERM_handler';
# Globals
our %default_options = (
my $default_options = {
verbose => 0,
debug => 0,
quiet => 0,
@@ -25,13 +21,18 @@ our %default_options = (
silent => 0,
help => 0,
pretend => 0,
job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
report_email => '',
config_file => '',
);
our %options = map { $_ => undef } keys %default_options;
job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
email_target => '',
email_sender => '',
email_subject => 'Rip completed',
email_timezone => 'Europe/London',
email_html => 'rip-completed.html',
email_text => 'rip-completed.txt',
};
my $options = { map { $_ => undef } keys %$default_options };
our %rip_options = (
my $rip_options = {
nice => 15,
input_filename => '/dev/sr0',
output_filename => 'rip-output.mkv',
@@ -46,56 +47,57 @@ our %rip_options = (
audio_codec => 'ac3',
audio_names => 'English',
subtitle_tracks => 1,
);
};
Getopt::Long::Configure( qw(bundling no_getopt_compat) );
GetOptions(
'verbose|v+' => \$options{verbose},
'debug|d' => \$options{debug},
'quiet|q' => \$options{quiet},
'silent|s' => \$options{silent},
'log|l=s' => \$options{log_file},
'help|h' => \$options{help},
'pretend|n' => \$options{pretend},
'job-servers|j=s@' => \$options{job_servers},
'report|r=s' => \$options{report_email},
'config|c=s' => \$options{config_file},
'nice|N=i' => \$rip_options{nice},
'input|i=s' => \$rip_options{input_filename},
'output|o=s' => \$rip_options{output_filename},
'title|t=s' => \$rip_options{title},
'format|f=s' => \$rip_options{format},
'video-encoder|e=s' => \$rip_options{video_codec},
'width|w=i' => \$rip_options{video_width},
'height|l=i' => \$rip_options{video_height},
'quantizer|Q=f' => \$rip_options{quantizer},
'deinterlate|D=i' => \$rip_options{deinterlace},
'audio-tracks|a=s' => \$rip_options{audio_tracks},
'audio-encoder|E=s' => \$rip_options{audio_codec},
'audio-name|A=s' => \$rip_options{audio_names},
'subtitle-tracks|S=s' => \$rip_options{subtitle_tracks},
'verbose|v+' => \$options->{verbose},
'debug|d' => \$options->{debug},
'quiet|q' => \$options->{quiet},
'silent|s' => \$options->{silent},
'log|l=s' => \$options->{log_file},
'help|h' => \$options->{help},
'pretend|n' => \$options->{pretend},
'job-servers|j=s@' => \$options->{job_servers},
'config|c=s' => \$options->{config_file},
'nice|N=i' => \$rip_options->{nice},
'input|i=s' => \$rip_options->{input_filename},
'output|o=s' => \$rip_options->{output_filename},
'title|t=s' => \$rip_options->{title},
'format|f=s' => \$rip_options->{format},
'video-encoder|e=s' => \$rip_options->{video_codec},
'width|w=i' => \$rip_options->{video_width},
'height|l=i' => \$rip_options->{video_height},
'quantizer|Q=f' => \$rip_options->{quantizer},
'deinterlate|D=i' => \$rip_options->{deinterlace},
'audio-tracks|a=s' => \$rip_options->{audio_tracks},
'audio-encoder|E=s' => \$rip_options->{audio_codec},
'audio-name|A=s' => \$rip_options->{audio_names},
'subtitle-tracks|S=s' => \$rip_options->{subtitle_tracks},
'email-target=s' => \$options->{email_target},
'email-sender=s' => \$options->{email_sender},
'email-subject=s' => \$options->{email_subject},
) or pod2usage(-verbose => 0);
pod2usage(-verbose => 1) if ($options{help});
pod2usage(-verbose => 1) if ($options->{help});
# Parse the configuration file (if any), and merge/validate the options
my $config = parse_config($options->{config_file});
($config, $options) = process_config($config, $options, $default_options);
# Create a list of jobs from the configuration file, and command line options
my @jobs;
my $config;
# Read a configuration file, if provided
if ($options{config_file}) {
$config = parse_config($options{config_file});
push @jobs, @{ $config->{jobs} };
}
# A list of jobs to run, formed from command line options or config file
if ($rip_options{title}) {
push @jobs, \%rip_options;
}
push @jobs, @{ $config->{jobs} } if defined $config->{jobs};
push @jobs, $rip_options if $options->{title};
die "No rips specified" unless @jobs;
# Setup logging
my $log = Log::Handler->new();
my $maxLogLevel = ($options{debug} ? 'debug' : ($options{quiet} ? 3 : $options{verbose} + 4)); # default to logging warning+, unless quiet in which case log error+, or debug in which case log everything
my $maxFileLogLevel = ($options{debug} ? 'debug' : 'info');
# default to logging warning+, unless quiet in which case log error+, or debug in which case log everything
my $maxLogLevel = ($options->{debug} ? 'debug' : ($options->{quiet} ? 3 : $options->{verbose} + 4));
# Ignore the quiet option when logging to disk
my $maxFileLogLevel = ($options->{debug} ? 'debug' : 'info');
if ( ! $options{silent}) {
if ( ! $options->{silent}) {
$log->add(
screen => {
log_to => 'STDOUT',
@@ -104,10 +106,10 @@ if ( ! $options{silent}) {
},
);
}
if ( $options{log_file}) {
if ( $options->{log_file}) {
$log->add(
file => {
filename => $options{log_file},
filename => $options->{log_file},
minlevel => 'emergency',
maxlevel => $maxFileLogLevel,
},
@@ -116,12 +118,12 @@ if ( $options{log_file}) {
# Setup the distribution client
my $client = Gearman::Client->new;
$client->job_servers($options{job_servers});
$client->job_servers($options->{job_servers});
# Add new ripping task for each job to run
my $taskset = $client->new_task_set;
foreach my $job (@jobs) {
$taskset->add_task("rip", freeze($job),
$taskset->add_task("handbrake_rip", freeze($job),
{
on_complete => \&on_complete_handler,
on_fail => \&on_fail_handler,
@@ -132,52 +134,49 @@ $taskset->wait;
sub on_complete_handler {
my $result_ref = shift or die;
my $response = thaw($$result_ref);
return on_fail_handler() unless defined $$result_ref;
if ($options{report_email}) {
if ($options->{email_target}) {
my $email = MIME::Lite::TT::HTML->new(
From => $options{report_email},,
To => $options{report_email},
Subject => 'Rip completed',
TimeZone => 'Europe/London',
From => $options->{email_sender},
To => $options->{email_target},
Subject => $options->{email_subject},
TimeZone => $options->{email_timezone},
Encoding => 'quoted-printable',
Template => {
html => 'rip-completed.html',
text => 'rip-completed.txt',
html => $options->{email_html},
text => $options->{email_text},
},
Charset => 'utf8',
TmplOptions => {},
TmplParams => {},
TmplOptions => {
INCLUDE_PATH => '.',
},
TmplParams => {
success => $response->{success},
filename => $response->{output_filename},
real_filename => $response->{real_output_filename},
log => join '\n', @{ $response->{log} },
},
);
$email->send;
}
$log->notice("Completed rip to $$result_ref");
$log->notice("Completed rip to $response->{real_output_filename}");
}
sub on_fail_handler {
$log->error("Rip failed");
$log->error("Failed to distribute job");
}
# Reads configuration options from a config file, expands the internal references, and returns the expanded form.
sub parse_config {
my $config_file = shift or die;
my $config = YAML::Any::LoadFile($options{config_file}) or die 'Unable to load configuration file: ' . $!;
my $config_file = shift;
# Update any unset options with values from config file
foreach my $option (keys %options) {
# Update the value of any option which has not been set by a command line argument
if (!$options{$option}) {
# Try the config file first, otherwise use the default value
if (defined $config->{options}->{$option}) {
$options{$option} = $config->{options}->{$option};
} else {
$options{$option} = $default_options{$option};
}
}
}
return {} unless defined $config_file;
my $config = YAML::Any::LoadFile($options->{config_file}) or die 'Unable to load configuration file: ' . $!;
# Iterate through each job, and inject any preset variables that haven't been redefined by the job
if (defined $config->{jobs}) {
foreach my $job (@{ $config->{jobs} }) {
if ($job->{presets}) {
foreach my $preset_name (@{ $job->{presets} }) {
@@ -187,19 +186,35 @@ sub parse_config {
}
}
}
}
return $config;
}
sub process_config {
my $config = shift or die;
my $options = shift or die;
my $default_options = shift or die;
sub INT_handler {
$log->error("Caught interrupt, exiting.");
exit 0;
# Update any unset options with values from config file
foreach my $option (keys %$options) {
# Update the value of any option which has not been set by a command line argument
if (!$options->{$option}) {
# Try the config file first, otherwise use the default value
if (defined $config->{options}->{$option}) {
$options->{$option} = $config->{options}->{$option};
} else {
$options->{$option} = $default_options->{$option};
}
}
}
sub TERM_handler {
$log->error("Caught SIGTERM, exiting.");
exit 0;
# Validate the email options
if ($options->{email_target}) {
$options->{email_sender} = $options->{email_target} unless $options->{email_sender};
}
return ($config, $options);
}
@@ -210,10 +225,30 @@ __END__;
=head1 SYNOPSIS
handbrake-cluster-client.pl -h
handbrake-cluster-client.pl [-v [-d]|-q|-s]
[-l logfile]
[-n]
handbrake-cluster-client.pl -h|--help
handbrake-cluster-client.pl [[-v|--verbose] | [-d|--debug] |
[-q|--quiet]] | [-s|--silent]]
[-l|--log <log file>]
[-n|--pretend]
[-c|--config <config file>]
[-N|--nice <nice level>]
[-i|--input <input filename>]
[-o|--output <output filename>]
[-f|--format <mkv|mp4>]
[-t|--title <title>]
[-e|--video-encoder <ffmpeg|theora|x264>]
[-w|--width <width>]
[-l|--height <height>]
[-Q|--quantizer <quality>]
[-D|--deinterlace <0|1|2>]
[-a|--audio-tracks <track list>]
[-E|--audio-encoder
<faac/lame/vorbis/ac3/dts>[,...]]
[-A|--audio-name <audio name list>]
[-s|--subtitle-tracks <subtitle track list>]
[--email-target <email address>]
[--email-sender <email address>]
[--email-subject <subject line>]
=head1 DESCRIPTION
@@ -223,4 +258,191 @@ __END__;
=head1 OPTIONS
=head2 Administrative Options
=over 4
=item B<-h|--help>
Displays this help information and quits.
=item B<-v|--verbose>
Displays verbose logging information.
=item B<-d|--debug>
Displays full debugging information, including all output from HandBrake.
=item B<-q|--quiet>
Displays only errors.
=item B<-s|--silent>
Displays no output whatsoever, useful for scripting. Output will still be
logged to disk if a file is specified.
=item B<-l|--log E<lt>log fileE<gt>>
Specifies the name of a file to write logging information to.
=item B<-n|--pretend>
Only shows what action would be taken, no rips are actually performed.
=item B<-j|--job-servers E<lt>server listE<gt>>
Specifies a comma separated list of alternate servers to use for job
distribution.
=item B<-c|--config E<lt>config fileE<gt>>
Specifies the name of a configuration file which contains additional options,
and a list of ripping jobs.
=item B<-n|--nice E<lt>nice levelE<gt>>
Specifies the priority to give the ripping process, using nice.
Defaults to 15.
=back
=head2 Ripping Options
=over 4
=item B<-i|--input E<lt>input filenameE<gt>>
Specifies the DVD drive device, or path to a VIDEO_TS folder to use as input
for the rip.
=item B<-o|--output E<lt>output filenameE<gt>>
Specifies the name of the file the rip will be saved to. This is only used
as a template; the actual rip will be saved to a filename with a unique
component at the end to avoid accidental overwrites.
=item B<-f|--format E<lt>mkv|mp4E<gt>>
Selects the container used for the rip. Defaults to mkv.
=item B<-t|--title E<lt>titleE<gt>>
Specifies the DVD title to rip. By default, the longest title will be used.
=item B<-e|--video-encoder E<lt>ffmpeg|theora|x264E<gt>>
Specifies the video encoder to use for the rip. Defaults to x264.
=item B<-w|--width E<lt>widthE<gt>>
Specifies the width of the output rip in pixels.
=item B<-l|--height E<lt>heightE<gt>>
Specifies the height of the output rip in pixels.
=item B<-Q|--quantizer E<lt>qualityE<gt>>
Specifies the x264 quantizer property, in the form of a real between 0 and 1.
Defaults to 0.61, aka "20".
=item B<D|--deinterlace E<lt>0|1|2E<gt>>
Specifies the type of deinterlacing to use. Use 0 to disable completely.
Use 1 for full deinterlacing, or 2 for selective deinterlacing (recommended).
Defaults to 0.
=item B<-a|--audio-tracks E<lt>audio track listE<gt>>
Specifies a comma separated list of audio tracks to include in the rip.
Defaults to 1.
=item B<-E|--audio-encoder E<lt>faac/lame/vorbis/ac3/dts>E<gt>>
Specifies the audio encoder to use for the rip. Defaults to ac3 pass-through.
=item B<-A|--audio-name E<lt>audio track namesE<gt>>
Specifies a comma separated list of names for the selected audio tracks.
Defaults to 'English'.
=back
=head2 Reporting Options
=over 4
=item B<--email-target E<lt>email addressE<gt>>
Specifies an email address to send a completion report to after each rip
finishes.
=item B<--email-sender E<lt>email addressE<gt>>
Specifies an email address to use as the sender of the rip completion email
reports. Defaults to the same as the target address.
=item B<--email-subject E<lt>subject lineE<gt>>
Overrides the subject line to use in the rip completion email reports.
=back
=head1 CONFIGURATION
This script accepts a configuration file in YAML format to specify multiple
ripping tasks, or provide values for commonly used options.
Command line arguments take precedence to options specified in config files.
This file is split into three sections:
=over 4
=item B<options>
Values for command-line options from the I<Administrative Options> section
above can be specified here, using variables with the same name as the long
options (with hyphens replaced with underscores).
options:
job_servers: build0.example.com
log_file: /tmp/handbrake.log
=item B<presets>
Presets allow the grouping of common job configuration options, for example
the input filename when ripping multiple episodes from a single DVD image.
Any option from the I<Ripping Options> section can be specified in a preset,
and options specified in a job take precedence in case of a clash.
preset:
disk1:
input_filename: /tmp/disk1/VIDEO_TS
tvseries:
format: mkv
video_codec: x264
=item B<jobs>
Specifies a list of ripping tasks to perform. Variables from the
I<Ripping Options> section above can be used here.
Additionally, you can include all of the options from one or more presets.
jobs:
- presets:
- disk1
- tvseries
output_filename: episode1.mkv
=back
=head2 EXAMPLE CONFIGURATION
An example configuration file is included in the distribution as C<config.yml>.
Refer to this when writing your own files.
=cut

112
handbrake-cluster-worker.pl
View File

@@ -10,13 +10,9 @@ use IPC::Open2;
use Log::Handler;
use Pod::Usage;
use String::Random qw/random_string/;
use Storable qw/thaw/;
use Storable qw/freeze thaw/;
use Switch;
# Handle interrupts, and term signals
$SIG{'INT'} = 'INT_handler';
$SIG{'TERM'} = 'TERM_handler';
# Globals
our %options = (
verbose => 0,
@@ -31,12 +27,12 @@ our %options = (
Getopt::Long::Configure( qw(bundling no_getopt_compat) );
GetOptions(
'verbose|v+' => \$options{verbose},
'help|h' => \$options{help},
'verbose|v' => \$options{verbose},
'debug|d' => \$options{debug},
'quiet|q' => \$options{quiet},
'silent|s' => \$options{silent},
'log|l=s' => \$options{log_file},
'help|h' => \$options{help},
'pretend|n' => \$options{pretend},
'job-servers|j=s@' => \$options{job_servers},
'handbrake' => \$options{handbrake},
@@ -69,19 +65,22 @@ if ( $options{log_file}) {
# Setup the worker, and listen for jobs
my $worker = Gearman::Worker->new(job_servers => $options{job_servers});
$worker->register_function(rip => \&do_rip);
$worker->register_function(handbrake_rip => \&handbrake_rip);
$worker->work while 1;
sub do_rip {
sub handbrake_rip {
my $job = shift;
my %rip_options = %{ thaw($job->arg) };
my %response;
$log->notice("Beginning new rip to $rip_options{output_filename}");
# Generate a unique filename based on the output filename to prevent clashes from previous runs
my $uuid = random_string('cccccc');
$log->debug("Using $uuid as unique identifier for this job");
$rip_options{unique_output_filename} = $rip_options{output_filename};
$rip_options{unique_output_filename} =~ s/\.([^\.]+)$/\.$uuid\.$1/;
$rip_options{real_output_filename} = $rip_options{output_filename};
$rip_options{real_output_filename} =~ s/\.([^\.]+)$/\.$uuid\.$1/;
$response{real_output_filename} = $rip_options{real_output_filename};
# Generate the command line for handbrake
my @handbrake_cmd = (
@@ -91,7 +90,7 @@ sub do_rip {
# Construct the handbrake command line
$options{handbrake},
get_options(\%rip_options, 'input_filename', '-i'),
get_options(\%rip_options, 'unique_output_filename', '-o'),
get_options(\%rip_options, 'real_output_filename', '-o'),
get_options(\%rip_options, 'title'),
get_options(\%rip_options, 'format', '-f'),
get_options(\%rip_options, 'video_codec', '-e'),
@@ -105,31 +104,36 @@ sub do_rip {
get_options(\%rip_options, 'subtitle_tracks', '-s'),
);
$log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd));
# Return a copy of the command used to rip the title
$response{handbrake_cmd} = @handbrake_cmd;
# Execute the ripping process
$log->debug("Beginning ripping process with command:\n" . join(' ', @handbrake_cmd));
my ($child_in, $child_out);
my $child_pid = open2($child_out, $child_in, @handbrake_cmd);
# Don't need to write from the child
# No need to write to the child process
close($child_in);
# Log the output from handbrake, and return it back to the client
$response{log} = ();
my $line;
while ($line = <$child_out>) {
push @{ $response{log} }, $line;
$log->debug($line);
}
close($child_out);
# If the rip process failed, report an error status here
waitpid($child_pid, 0);
my $child_exit_status = $? >> 8;
if ($child_exit_status) {
$log->warning("Ripping process returned error status: $child_exit_status");
return undef;
$response{status} = $? >> 8;
$response{success} = $response{status} == 0;
if ($response{success}) {
$log->notice("Finished rip to $rip_options{real_output_filename}");
} else {
$log->warning("Ripping process returned error status: $response{status}");
}
$log->notice("Finished rip to $rip_options{unique_output_filename}");
return $rip_options{unique_output_filename};
return freeze(\%response);
}
sub get_options {
@@ -155,17 +159,6 @@ sub get_options {
}
}
sub INT_handler {
$log->error("Caught interrupt, exiting.");
exit 0;
}
sub TERM_handler {
$log->error("Caught SIGTERM, exiting.");
exit 0;
}
__END__;
=head1 NAME
@@ -173,10 +166,14 @@ __END__;
=head1 SYNOPSIS
handbrake-cluster-worker.pl -h
handbrake-cluster-worker.pl [-v [-d]|-q|-s]
[-l logfile]
[-n]
handbrake-cluster-worker.pl -h|--help
handbrake-cluster-worker.pl [[-v|--verbose] | [-d|--debug] |
[-q|--quiet] | [-s|--silent]]
[-l|--log <log file>]
[-n|--pretend]
[-j|--job-servers <server list>]
[--handbrake <handbrake executable>]
=head1 DESCRIPTION
@@ -185,4 +182,45 @@ __END__;
=head1 OPTIONS
=over 4
=item B<-h|--help>
Displays this help information and quits.
=item B<-v|--verbose>
Displays verbose logging information.
=item B<-d|--debug>
Displays full debugging information, including all output from HandBrake.
=item B<-q|--quiet>
Displays only errors.
=item B<-s|--silent>
Displays no output whatsoever, useful for scripting. Output will still be
logged to disk if a file is specified.
=item B<-l|--log E<lt>log fileE<gt>>
Specifies the name of a file to write logging information to.
=item B<-n|--pretend>
Only shows what action would be taken, no rips are actually performed.
=item B<-j|--job-servers E<lt>server listE<gt>>
Specifies a comma separated list of alternate servers to use for job
distribution.
=item B<--handbrake E<lt>handbrake executableE<gt>>
Specifies an alternate HandBrake executable to use. Default is
C</usr/bin/HandBrakeCLI>.
=cut

10
rip-completed.html
View File

@@ -1 +1,9 @@
A rip has completed successfully
[% IF success %]
A HandBrake rip to [% real_filename %] has completed successfully.
[% ELSE %]
A HandBrake rip to [% real_filename %] has failed.
[% END %]
A log of the activity is included below.
[% log %]

10
rip-completed.txt
View File

@@ -1 +1,9 @@
A rip has completed successfully
[% IF success %]
A HandBrake rip to [% real_filename %] has completed successfully.
[% ELSE %]
A HandBrake rip to [% real_filename %] has failed.
[% END %]
A log of the activity is included below.
[% log %]