diff options
Diffstat (limited to 'emi')
| -rwxr-xr-x | emi | 158 |
1 files changed, 2 insertions, 156 deletions
@@ -34,6 +34,7 @@ use Iurt::Process qw(check_pid); use Iurt::Mail qw(sendmail); use Iurt::File qw(get_upload_tree_state); use Iurt::Util qw(plog_init plog); +use Iurt::Emi qw(find_prefixes_ready_to_upload upload_prefix_in_media); use Data::Dumper; use MDK::Common qw(cat_ if_ find touch); use MDK::Common::DataStructure qw(difference2); @@ -119,162 +120,7 @@ my $done = "$config->{queue}/done"; my $reject = "$config->{queue}/rejected"; my %pkg_tree = get_upload_tree_state($config); - -# -# Decide what should be uploaded -# - -# $targets{$target}{$media}{arch_finisher}{$arch}: prefix on which we need to actions to get this arch updated -# $targets{$target}{$media}{to_upload}: list of prefixes to upload -my %targets; - -foreach my $prefix (sort keys %pkg_tree) { - my $target = $pkg_tree{$prefix}{target}; - my $mandatory_arch = get_mandatory_arch($config, $target); - - plog('NOTIFY', "processing $prefix"); - - my $ok = 1; - - foreach my $media (keys %{$pkg_tree{$prefix}{media}}) { - my @wanted_archs = defined($pkg_tree{$prefix}{media}{$media}{arch}{noarch}) ? 'noarch' : @$mandatory_arch; - my $path = $pkg_tree{$prefix}{media}{$media}{path}; - my %missing; - - plog('DEBUG', "... in $path"); - - if ($pkg_tree{$prefix}{media}{$media}{uploaded}) { - plog('INFO', "package already uploaded for mandatory arches, proceeding"); - next; - } - - foreach my $m (@wanted_archs, 'src') { - $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$m} and next; - my $x = "yes"; - if (!$pkg_tree{$prefix}{media}{$media}{arch}{$m}) { - $missing{$m} = 1; - $x = "no"; - $ok = 0; - } - plog('INFO', " mandatory architecture $m present: $x"); - } - - unless ($ok) { - plog('INFO', "mandatory arch", join(' ', keys %missing), - "missing for $media, waiting"); - next; - } - } - next unless $ok; - - # - # All mandatory archs found, mark for upload - # - foreach my $media (keys %{$pkg_tree{$prefix}{media}}) { - $targets{$target}{$media} ||= { 'arch_finisher' => {}, 'is_finisher' => {}, 'to_upload' => [] }; - - push @{$targets{$target}{$media}{to_upload}}, $prefix; - - # We already have found universal finisher in that media, we're fine - next if exists $targets{$target}{$media}{arch_finisher}{noarch}; - - if ($pkg_tree{$prefix}{media}{$media}{arch}{noarch}) { - # This package is noarch, genhdlist for it will touch all archs - $targets{$target}{$media}{arch_finisher} = { 'noarch' => $prefix }; - } else { - my $has_new_arch = scalar(difference2([ keys %{$pkg_tree{$prefix}{media}{$media}{arch}} ], [ keys %{$targets{$target}{$media}{arch_finisher}} ])); - if ($has_new_arch) { - # We need this package to cover the new arch - # Set it for all, it may allow getting rid of some others - foreach (keys %{$pkg_tree{$prefix}{media}{$media}{arch}}) { - $targets{$target}{$media}{arch_finisher}{$_} = $prefix; - } - } - } - } -} - -sub upload_prefix_in_media { - my ($prefix, $media, $o_finish) = @_; - my (@packages, @duplicate_packages); - my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+$/; - my $target = $pkg_tree{$prefix}{target}; - my $youri_file = "$prefix.youri"; - if ($pkg_tree{$prefix}{media}{$media}{uploaded}) { - $youri_file .= "." . time(); - } - - my $path = $pkg_tree{$prefix}{media}{$media}{path}; - - plog('OK', "all mandatory archs done: $prefix"); - foreach my $rpm (@{$pkg_tree{$prefix}{media}{$media}{rpms}}) { - my $rpmpath = "$done/$path/${prefix}_$rpm"; - if ($pkg_tree{$prefix}{media}{$media}{uploaded}) { - # if already uploaded for mandatory arches, do not try to upload again src or noarch packages - # but still remember these duplicate files for removal - if (my ($type) = $rpm =~ /\.(noarch|src)\.rpm$/) { - plog('DEBUG', "$prefix already uploaded for mandatory arches, not re-uploading $type $rpm"); - push @duplicate_packages, $rpmpath; - next; - } - } - push @packages, $rpmpath; - plog('OK', " uploading $rpm in $done/$path"); - } - - $user ||= $config->{upload_user}; - # FIXME we want to skip all post, we should not hardcode them here - my $skip = $o_finish ? "" : "--skip-post genhdlist2 --skip-post mirror --skip-post clean_rpmsrate"; - my $command = "/usr/bin/perl -I/usr/share/mga-youri-submit/lib /usr/share/mga-youri-submit/bin/youri-submit --verbose --config /etc/youri/submit-upload.conf --define user=$user --define prefix=$prefix --define section=$media $skip $target @packages &> $done/$path/$youri_file"; - - plog('DEBUG', "running $command"); - if (!system($command)) { - plog('INFO', "upload succeeded"); - } else { - # should send a mail or something - plog('ERROR', "upload failed ($!), rejecting files in $reject/$path/"); - make_path("$reject/$path"); - foreach my $rpm (@{$pkg_tree{$prefix}{media}{$media}{rpms}}) { - link("$done/$path/${prefix}_$rpm", "$reject/$path/${prefix}_$rpm") or plog('ERROR', "ERROR: link of $rpm failed ($!)"); - } - link("$done/$path/$youri_file", "$reject/$path/$youri_file"); - - my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+/; - if ($user) { - my @pkgs = grep { !/src\.rpm$/ } @{$pkg_tree{$prefix}{media}{$media}{rpms}}; - my $text = join("\n", qq(The upload of the following packages failed:\n), map { "- $_" } @pkgs) . "\n"; - my $rpms = join(' ', @pkgs, undef); - my $to = get_author_email($user) || "Unknown <$config->{admin}>"; - $text .= "\nUpload log available in $config->{http_queue}/rejected/$path/$youri_file\n"; - - sendmail($to, undef, "Upload failed for $rpms", $text, "Emi the upload bot <$config->{admin}>", 0, $config); - } - - # should delete the files - } - - # delete the files which should have heen either put in queue or rejected - unlink(@packages, @duplicate_packages); - - # unlink the sources rpm unless some non mandatory arch still need to be done - my $all_done = 1; - if (!defined($pkg_tree{$prefix}{media}{$media}{arch}{noarch})) { - my $arch_list = find { ref($_) eq 'ARRAY' } $config->{arch}, (ref($config->{arch}) eq 'HASH' ? ($config->{arch}{$target}, $config->{arch}{default}) : ()); - my @arch_list = $arch_list ? @$arch_list : keys %{$config->{bot}}; - # If we are here, mandatory arches are done, no need to check them - my $mandatory_arch = get_mandatory_arch($config, $target); - foreach my $arch (difference2(\@arch_list, $mandatory_arch)) { - next if $pkg_tree{$prefix}{media}{$media}{arch}{$arch}; - $all_done = 0; - } - } - if ($all_done) { - foreach (@{$pkg_tree{$prefix}{srpms}}) { - plog('DEBUG', "unlink $todo/$path/${prefix}_$_"); - unlink("$todo/$path/${prefix}_$_"); - } - } -} +my %targets = find_prefixes_ready_to_upload($config, %pkg_tree); foreach my $target (keys %targets) { foreach my $media (keys %{$targets{$target}}) { |