diff options
Diffstat (limited to 'emi')
| -rwxr-xr-x | emi | 278 |
1 files changed, 171 insertions, 107 deletions
@@ -46,35 +46,65 @@ my $HOME = $ENV{HOME}; my $configfile = "$HOME/.upload.conf"; my %config_usage = ( - admin => { desc => 'mail address of the bot administrator', default => 'warly@mandriva.com' }, + admin => { + desc => 'mail address of the bot administrator', + default => 'warly@mandriva.com' + }, arch => { desc => "List of arch", - default => [ 'i586', 'x86_64', 'ppc' , 'sparc', 'sparc64' ], - }, - default_mail_domain => { desc => "Default mail domain to append", default => 'mandriva.org' }, - http_queue => { desc => 'Address where log can be consulted', default => 'http://svn.mandriva.com/queue/'}, + default => [ 'i586', 'x86_64', 'ppc' , 'sparc', 'sparc64' ] + }, + default_mail_domain => { + desc => "Default mail domain to append", + default => 'mandriva.org' + }, + http_queue => { + desc => 'Address where log can be consulted', + default => 'http://svn.mandriva.com/queue/' + }, mandatory_arch => { desc => 'List of mandatory architecture to be able to upload', - default => [ 'i586', 'x86_64' ], - }, - tmp => { desc => "Temporary directory", default => "$HOME/tmp/"}, - root => { desc => 'Architecture root dir', default => "/mnt/BIG/dis/" }, - cache_home => { desc => 'Where to store the cache files', default => "$HOME/.bugs" }, - upload_user => { desc => 'User who is uploading packages', default => 'mandrake' }, - repsys_conf => { desc => 'Path of the repsys.conf file which includes login mail corresponding', default => '/etc/repsys.conf' }, - queue => { desc => 'root directory of the various upload queues', default => '/home/mandrake/uploads/'}, - ssh_option => { desc => "SSH options", default => "-o ConectTimeout=20" }, + default => [ 'i586', 'x86_64' ] + }, + tmp => { + desc => "Temporary directory", + default => "$HOME/tmp/" + }, + root => { + desc => 'Architecture root dir', + default => "/mnt/BIG/dis/" + }, + cache_home => { + desc => 'Where to store the cache files', + default => "$HOME/.bugs" + }, + upload_user => { + desc => 'User who is uploading packages', + default => 'mandrake' + }, + repsys_conf => { + desc => 'Path of repsys.conf file with login mail corresponding', + default => '/etc/repsys.conf' + }, + queue => { + desc => 'root directory of the various upload queues', + default => '/home/mandrake/uploads/' + }, + ssh_option => { + desc => "SSH options", + default => "-o ConectTimeout=20" + }, ); my $config; if (-f $configfile) { - $config = do $configfile or die "FATAL $program_name: syntax error in $configfile"; + $config = do $configfile + or die "FATAL $program_name: syntax error in $configfile"; } else { $config = {} } config_usage(\%config_usage, $config) if $run{config_usage}; - config_init(\%config_usage, $config, \%run); my $mail = get_repsys_conf($config->{repsys_conf}); @@ -88,7 +118,6 @@ my $cache = { arch => {} }; my ($fulldate, $daydate) = get_date(); $run{daydate} = $daydate; -print {$run{LOG}} "$program_name\n"; my $todo = "$config->{queue}/todo/"; my $done = "$config->{queue}/done/"; @@ -96,121 +125,156 @@ my $reject = "$config->{queue}/rejected/"; my %pkg_tree; my %excluded; -check_upload_tree(\%run, $done, - sub { - my ($todo, $f, $m, $s, $r) = @_; - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) { - my ($prefix, $rpm, $arch) = ($1, $2, $3); - print {$run{LOG}} "$program_name: found rpm $rpm ($prefix)\n"; - $pkg_tree{$prefix}{path} = "/$f/$m/$s"; - $pkg_tree{$prefix}{arch}{$arch} = 1; - $pkg_tree{$prefix}{target} = $f; - $pkg_tree{$prefix}{section} = "$m/$s"; - push @{$pkg_tree{$prefix}{srpms}}, $rpm if ($arch eq 'src'); - push @{$pkg_tree{$prefix}{rpms}} , $rpm; - } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.done$/) { - my ($prefix, $arch) = ($1, $2); - $cache->{arch}{$prefix}{$arch} = 1 - } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.excluded$/) { - my ($prefix, $arch) = ($1, $2); - $cache->{arch}{$prefix}{$arch} = 1; - $excluded{$prefix}{$arch} = 1 - } - }, -); -check_upload_tree(\%run, $todo, - sub { - my ($todo, $f, $m, $s, $r) = @_; - if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) { - my ($prefix, $rpm, $arch) = ($1, $2, $3); - print {$run{LOG}} "$program_name: found todo rpm $rpm ($prefix)\n"; - push @{$pkg_tree{$prefix}{todo}}, $rpm - } - }, -); +# +# Gather data from upload tree +# + +sub done_func { + my ($todo, $f, $m, $s, $r) = @_; + + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) { + my ($prefix, $rpm, $arch) = ($1, $2, $3); + plog("found rpm $rpm ($prefix)"); + $pkg_tree{$prefix}{path} = "/$f/$m/$s"; + $pkg_tree{$prefix}{arch}{$arch} = 1; + $pkg_tree{$prefix}{target} = $f; + $pkg_tree{$prefix}{section} = "$m/$s"; + push @{$pkg_tree{$prefix}{srpms}}, $rpm if ($arch eq 'src'); + push @{$pkg_tree{$prefix}{rpms}} , $rpm; + } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.done$/) { + my ($prefix, $arch) = ($1, $2); + #plog("found .done ($prefix)"); + $cache->{arch}{$prefix}{$arch} = 1; + } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.excluded$/) { + my ($prefix, $arch) = ($1, $2); + plog("found .excluded ($prefix)"); + $cache->{arch}{$prefix}{$arch} = 1; + $excluded{$prefix}{$arch} = 1; + } +} + +sub todo_func { + my ($todo, $f, $m, $s, $r) = @_; + + if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) { + my ($prefix, $rpm, $arch) = ($1, $2, $3); + plog("$program_name: found todo rpm $rpm ($prefix)"); + push @{$pkg_tree{$prefix}{todo}}, $rpm; + } +} + +check_upload_tree(\%run, $done, \&done_func, ); +check_upload_tree(\%run, $todo, \&todo_func, ); + # Once the tree is checked, ulri can be launched again unlink $pidfile; + +# +# Decide what should be uploaded +# + foreach my $prefix (keys %pkg_tree) { my $ok = 1; my $target = $pkg_tree{$prefix}{target}; my $path = $pkg_tree{$prefix}{path}; my $section = $pkg_tree{$prefix}{section}; - print {$run{LOG}} "$program_name: processing $prefix in $path\n"; + + plog("processing $prefix in $path"); my %missing; foreach my $mandatory_arch (@{$config->{mandatory_arch}}, 'src') { $excluded{$prefix}{$mandatory_arch} and next; - print {$run{LOG}} "$program_name: checking if mandatory architecture $mandatory_arch is present: "; + my $x = "yes"; if (!$pkg_tree{$prefix}{arch}{$mandatory_arch} && !$pkg_tree{$prefix}{arch}{noarch}){ if (!$cache->{arch}{$prefix}{$mandatory_arch}) { $missing{$mandatory_arch} = 1; - print {$run{LOG}} "no\n"; - $ok = 0 + $x = "no"; + $ok = 0; } else { - print {$run{LOG}} "yes (in cache)\n" + $x = "yes (in cache)"; } - } else { - print {$run{LOG}} "yes\n" } + plog(" mandatory architecture $mandatory_arch present: $x"); + } + + unless ($ok) { + plog(0, "WARNING: mandatory arch", join(' ', keys %missing), + "missing, waiting"); + next; + } + + # + # All mandatory archs found, upload + # + + my @packages; + my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+$/; + + foreach my $rpm (@{$pkg_tree{$prefix}{rpms}}) { + push @packages, "$done/$path/${prefix}_$rpm" } - if ($ok) { - # upload - my @packages; - my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+$/; + + $user ||= $config->{upload_user}; + + my $command = "/usr/bin/sudo -H -u $config->{upload_user} /usr/bin/perl -I/usr/local/lib/perl/ /usr/local/bin/youri-submit --verbose --config /etc/youri/submit-queue.conf --define user=$user --define prefix=$prefix --define section=$section $target @packages &> $done/$path/$prefix.youri"; + + plog("running $command"); + if (!system($command)) { + plog("upload succeeded."); + + # now check if we need to keep the current srpm in todo + my $all_uploaded = 1; + + foreach my $arch (@{$config->{arch}}) { + if (!$cache->{arch}{$prefix}{$arch}) { + $all_uploaded = 0; + } + } + + if ($all_uploaded) { + plog("cleaning upload tree for $prefix"); + # remove srpm + # remove lock + } + } else { + # should send a mail or something + plog("upload failed ($!), rejecting files in $reject/$path/"); foreach my $rpm (@{$pkg_tree{$prefix}{rpms}}) { - push @packages, "$done/$path/${prefix}_$rpm" + my $err = link "$done/$path/${prefix}_$rpm", "$reject/$path/${prefix}_$rpm"; + plog("ERROR: link of $rpm failed ($!)"); } - $user ||= $config->{upload_user}; - my $command = "/usr/bin/sudo -H -u $config->{upload_user} /usr/bin/perl -I/usr/local/lib/perl/ /usr/local/bin/youri-submit --verbose --config /etc/youri/submit-queue.conf --define user=$user --define prefix=$prefix --define section=$section $target @packages &> $done/$path/$prefix.youri"; - print {$run{LOG}} "$program_name: running $command\n"; - if (!system($command)) { - print {$run{LOG}} "$program_name: upload succeeded.\n"; - # now check if we need to keep the current srpm in todo - my $all_uploaded = 1; - foreach my $arch (@{$config->{arch}}) { - if (!$cache->{arch}{$prefix}{$arch}) { - $all_uploaded = 0 - } - } - if ($all_uploaded) { - print {$run{LOG}} "$program_name: cleaning upload tree for $prefix\n"; - # remove srpm - # remove lock - } - } else { - # should send a mail or something - print {$run{LOG}} "$program_name: upload failed ($!), rejecting files in $reject/$path/\n"; + link "$done/$path/$prefix.youri", "$reject/$path/$prefix.youri"; + + my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+/; + if ($user) { + my $text = qq{The upload of the following packages failed:\n}; + my $rpms; foreach my $rpm (@{$pkg_tree{$prefix}{rpms}}) { - my $err = link "$done/$path/${prefix}_$rpm", "$reject/$path/${prefix}_$rpm"; - print {$run{LOG}} "ERROR $program_name: link of $rpm failed ($!)\n" + $rpm =~ /src\.rpm$/ or next; + $rpms .= "$rpm "; + $text .= "- $rpm\n" } - link "$done/$path/$prefix.youri", "$reject/$path/$prefix.youri"; - my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+/; - if ($user) { - my $text = qq{The upload of the following packages failed:\n}; - my $rpms; - foreach my $rpm (@{$pkg_tree{$prefix}{rpms}}) { - $rpm =~ /src\.rpm$/ or next; - $rpms .= "$rpm "; - $text .= "- $rpm\n" - } - my $to = $mail->{$user} || "$user\@mandriva.com, $user\@mandriva.org"; - $text .= "\nUpload log available in $config->{http_queue}/rejected/$path/$prefix.youri\n"; - sendmail($to, $config->{admin} , "Upload failed for $rpms", $text, "Emi the upload bot <$config->{admin}>", 0) - } - # should delete the files + my $to = $mail->{$user} || "$user\@mandriva.com, $user\@mandriva.org"; + $text .= "\nUpload log available in $config->{http_queue}/rejected/$path/$prefix.youri\n"; + sendmail($to, $config->{admin} , "Upload failed for $rpms", $text, "Emi the upload bot <$config->{admin}>", 0) } - # delete the files which should have heen either put in queue or rejected - unlink $_ foreach @packages; - # keep the log file for debugging - # unlink "$done/$path/$prefix.youri"; - # unlink the sources rpm, other arch will be able to grab it into the repository - print "unlink $todo/$path/${prefix}_$_\n" foreach @{$pkg_tree{$prefix}{todo}}; - unlink "$todo/$path/${prefix}_$_" foreach @{$pkg_tree{$prefix}{todo}} - } else { - print {$run{LOG}} "WARNING $program_name: mandatory arch (", join(' ', keys %missing),") are missing for $prefix, waiting\n" + + # should delete the files + } + + # delete the files which should have heen either put in queue or rejected + unlink $_ foreach @packages; + + # keep the log file for debugging + unlink "$done/$path/$prefix.youri"; + + # unlink the sources rpm, other arch will be able to grab it into + # the repository + foreach (@{$pkg_tree{$prefix}{todo}}) { + plog("unlink $todo/$path/${prefix}_$_"); + unlink "$todo/$path/${prefix}_$_"; } } |