ben/handbrake-cluster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added identifiers to jobs

Browse Source 
Updated config file to specify ids for jobs for progress and logging
Added default_rip_options, which are copied to all rips without
specified values
Added an optional input/output_dir variables to prefix
input/output_filename
This commit is contained in:
Ben Roberts
2010-02-23 21:36:56 +00:00
parent f4e038951d
commit 9358c100ae
2 changed files with 49 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
handbrake-cluster-client.pl
View File

@@ -23,6 +23,7 @@ my $default_options = {
pretend => 0,
config_file => '',
job_servers => ['build0.sihnon.net', 'build1.sihnon.net', 'build2.sihnon.net'],
limit => [],
email_target => '',
email_sender => '',
email_subject => 'Rip completed',
@@ -32,15 +33,17 @@ my $default_options = {
};
my $options = { map { $_ => undef } keys %$default_options };
my $rip_options = {
my $default_rip_options = {
nice => 15,
input_dir => '',
input_filename => '/dev/sr0',
output_dir => '',
output_filename => 'rip-output.mkv',
title => 0,
format => 'mkv',
video_codec => 'x264',
video_width => 720, # DVD resolution
video_height => 576, # DVD resolution
video_width => 0,
video_height => 0,
quantizer => 0.61,# x264 quantizer = 20
deinterlace => 0, # 0 = off, 1 = on, 2 = selective
audio_tracks => 1,
@@ -48,6 +51,7 @@ my $rip_options = {
audio_names => 'English',
subtitle_tracks => 1,
};
my $rip_options = { map { $_ => undef } keys %$default_rip_options };
Getopt::Long::Configure( qw(bundling no_getopt_compat) );
GetOptions(
@@ -60,6 +64,7 @@ GetOptions(
'pretend|n' => \$options->{pretend},
'job-servers|j=s@' => \$options->{job_servers},
'config|c=s' => \$options->{config_file},
'limit|L=s' => sub { my ($name, $value) = @_; $options->{limit} = [split(/,/, $value)]; },
'nice|N=i' => \$rip_options->{nice},
'input|i=s' => \$rip_options->{input_filename},
'output|o=s' => \$rip_options->{output_filename},
@@ -81,14 +86,13 @@ GetOptions(
pod2usage(-verbose => 1) if ($options->{help});
# Parse the configuration file (if any), and merge/validate the options
my $config = parse_config($options->{config_file});
my $config = parse_config($options->{config_file}, $default_rip_options);
($config, $options) = process_config($config, $options, $default_options);
# Create a list of jobs from the configuration file, and command line options
my @jobs;
push @jobs, @{ $config->{jobs} } if defined $config->{jobs};
push @jobs, $rip_options if $options->{title};
die "No rips specified" unless @jobs;
my %jobs = %{ $config->{jobs} } if defined $config->{jobs};
$jobs{__commandline} = $rip_options if $options->{title};
die "No rips specified" unless %jobs;
# Setup logging
my $log = Log::Handler->new();
@@ -121,11 +125,15 @@ my $client = Gearman::Client->new;
$client->job_servers($options->{job_servers});
# Add new ripping task for each job to run
my @progress;
my %progress;
my $taskset = $client->new_task_set;
foreach my $i (0..$#jobs) {
my $job = $jobs[$i];
$progress[$i] = 0;
foreach my $job_id (keys %jobs) {
my $job = $jobs{$job_id};
# Check that the job hasn't been restricted by the limit option
next if (!$options->{limit} || !$options->{job_permitted}->{$job_id});
$progress{$job_id} = 0;
$taskset->add_task('handbrake_rip', freeze($job),
{
@@ -133,20 +141,21 @@ foreach my $i (0..$#jobs) {
my $numerator = shift;
my $denominator = shift or die;
$progress[$i] = $numerator / $denominator;
$progress{$job_id} = $numerator / $denominator;
display_progress(@progress);
display_progress(%progress);
},
on_complete => \&on_complete_handler,
on_retry => sub {
my $attempt = shift or die;
$log->warning("Retrying rip");
$log->warning("Retrying '$job_id' rip");
},
on_fail => sub {
$log->warning("Rip failed");
$log->warning("Rip '$job_id' failed");
},
}
);
$log->info("Enqueued '$job_id' rip");
}
$taskset->wait;
@@ -183,24 +192,28 @@ sub on_complete_handler {
}
sub display_progress {
my @progress = @_;
my %progress = @_;
local $|;
$| = 1;
print "Completion: " . join(' ', map { "$_($progress[$_]%)" } 0..$#progress) . "\r";
print "Completion: " . join(' ', map { "$_($progress{$_}%)" } keys %progress) . "\r";
}
# Reads configuration options from a config file, expands the internal references, and returns the expanded form.
sub parse_config {
my $config_file = shift;
my $default_rip_options = shift or die;
return {} unless defined $config_file;
my $config = YAML::Any::LoadFile($options->{config_file}) or die 'Unable to load configuration file: ' . $!;
# Iterate through each job, and inject any preset variables that haven't been redefined by the job
if (defined $config->{jobs}) {
foreach my $job (@{ $config->{jobs} }) {
foreach my $job_id (keys %{ $config->{jobs} }) {
my $job = $config->{jobs}->{$job_id};
$job = $config->{jobs}->{$job_id} = {} unless $job;
if ($job->{presets}) {
foreach my $preset_name (@{ $job->{presets} }) {
foreach my $preset_key (keys %{$config->{presets}->{$preset_name}}) {
@@ -208,6 +221,11 @@ sub parse_config {
}
}
}
# Inject any default variables that haven't been defined by the job or preset
foreach my $rip_option (keys %$default_rip_options) {
$job->{$rip_option} = $default_rip_options->{$rip_option} unless $job->{$rip_option};
}
}
}
@@ -232,6 +250,13 @@ sub process_config {
}
}
# Flag jobs that may be run if any limits are specified
my %job_permitted = map { $_ => 0 } keys %{ $config->{jobs} };
foreach my $job_id (@{ $options->{limit} }) {
$job_permitted{$job_id} = 1;
}
$options->{job_permitted} = \%job_permitted;
# Validate the email options
if ($options->{email_target}) {
$options->{email_sender} = $options->{email_target} unless $options->{email_sender};

6
handbrake-cluster-worker.pl
View File

@@ -81,7 +81,7 @@ sub handbrake_rip {
# Generate a unique filename based on the output filename to prevent clashes from previous runs
my $uuid = random_string('cccccc');
$rip_options->{real_output_filename} = $rip_options->{output_filename};
$rip_options->{real_output_filename} = $rip_options->{output_dir} . $rip_options->{output_filename};
$rip_options->{real_output_filename} =~ s/\.([^\.]+)$/\.$uuid\.$1/;
$response->{real_output_filename} = $rip_options->{real_output_filename};
@@ -196,6 +196,10 @@ sub get_options {
my $option = shift;
switch ($option_name) {
case 'input_filename' {
return ('-i', $rip_options->{input_dir} . $rip_options->{input_filename});
}
case 'title' {
return ('-L') if ! defined($rip_options->{$option_name}) || $rip_options->{$option_name} < 0;
return ('-t', $rip_options->{$option_name});