package Iurt::Emi; use base qw(Exporter); use File::Path qw(make_path); use Iurt::Config qw(get_author_email get_mandatory_arch); use Iurt::Mail qw(sendmail); use Iurt::Util qw(plog); use MDK::Common::Func qw(find); use MDK::Common::DataStructure qw(difference2); use strict; our @EXPORT = qw( find_prefixes_ready_to_upload upload_prefix_in_media ); sub find_prefixes_ready_to_upload { my ($config, %pkg_tree) = @_; # $targets{$target}{$media}{arch_finisher}{$arch}: prefix on which we need to actions to get this arch updated # $targets{$target}{$media}{to_upload}: list of prefixes to upload my %targets; foreach my $prefix (sort keys %pkg_tree) { my $target = $pkg_tree{$prefix}{target}; my $mandatory_arch = get_mandatory_arch($config, $target); plog('NOTIFY', "processing $prefix"); my $ok = 1; foreach my $media (keys %{$pkg_tree{$prefix}{media}}) { my @wanted_archs = defined($pkg_tree{$prefix}{media}{$media}{arch}{noarch}) ? 'noarch' : @$mandatory_arch; my $path = $pkg_tree{$prefix}{media}{$media}{path}; my %missing; plog('DEBUG', "... in $path"); if ($pkg_tree{$prefix}{media}{$media}{uploaded}) { plog('INFO', "package already uploaded for mandatory arches, proceeding"); next; } foreach my $m (@wanted_archs, 'src') { $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$m} and next; my $x = "yes"; if (!$pkg_tree{$prefix}{media}{$media}{arch}{$m}) { $missing{$m} = 1; $x = "no"; $ok = 0; } plog('INFO', " mandatory architecture $m present: $x"); } unless ($ok) { plog('INFO', "mandatory arch", join(' ', keys %missing), "missing for $media, waiting"); next; } } next unless $ok; # # All mandatory archs found, mark for upload # foreach my $media (keys %{$pkg_tree{$prefix}{media}}) { $targets{$target}{$media} ||= { 'arch_finisher' => {}, 'is_finisher' => {}, 'to_upload' => [] }; push @{$targets{$target}{$media}{to_upload}}, $prefix; # We already have found universal finisher in that media, we're fine next if exists $targets{$target}{$media}{arch_finisher}{noarch}; my %arches_to_upload = {}; foreach (@{$pkg_tree{$prefix}{media}{$media}{rpms}}) { next unless /\.([^.]*)\.rpm/; my $arch = $1; next if $arch eq 'src'; # If this is a secondary upload, noarch package will be dropped next if ($arch eq 'noarch' && $pkg_tree{$prefix}{media}{$media}{uploaded}); $arches_to_upload{$arch} = 1; } if ($arches_to_upload{noarch}) { # This package is noarch, genhdlist for it will touch all archs $targets{$target}{$media}{arch_finisher} = { 'noarch' => $prefix }; } else { my $has_new_arch = scalar(difference2([ keys %arches_to_upload ], [ keys %{$targets{$target}{$media}{arch_finisher}} ])); if ($has_new_arch) { # We need this package to cover the new arch # Set it for all, it may allow getting rid of some foreach (keys %arches_to_upload) { $targets{$target}{$media}{arch_finisher}{$_} = $prefix; } } } } } return %targets; } sub upload_prefix_in_media { my ($config, $pkg_tree, $prefix, $media, $o_finish) = @_; my $todo = "$config->{queue}/todo"; my $done = "$config->{queue}/done"; my $reject = "$config->{queue}/rejected"; my (@packages, @duplicate_packages); my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+$/; my $target = $pkg_tree->{$prefix}{target}; my $youri_file = "$prefix.youri"; if ($pkg_tree->{$prefix}{media}{$media}{uploaded}) { $youri_file .= "." . time(); } my $path = $pkg_tree->{$prefix}{media}{$media}{path}; plog('OK', "all mandatory archs done: $prefix"); foreach my $rpm (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) { my $rpmpath = "$done$path/${prefix}_$rpm"; if ($pkg_tree->{$prefix}{media}{$media}{uploaded}) { # if already uploaded for mandatory arches, do not try to upload again src or noarch packages # but still remember these duplicate files for removal if (my ($type) = $rpm =~ /\.(noarch|src)\.rpm$/) { plog('DEBUG', "$prefix already uploaded for mandatory arches, not re-uploading $type $rpm"); push @duplicate_packages, $rpmpath; next; } } push @packages, $rpmpath; plog('OK', " uploading $rpm in $done/$path"); } # This should not happen :( return unless @packages; $user ||= $config->{upload_user}; # FIXME we want to skip all post, we should not hardcode them here my $skip = $o_finish ? "" : "--skip-post genhdlist2 --skip-post mirror --skip-post clean_rpmsrate"; my $command = "/usr/bin/perl -I/usr/share/mga-youri-submit/lib /usr/share/mga-youri-submit/bin/youri-submit --verbose --config /etc/youri/submit-upload.conf --define user=$user --define prefix=$prefix --define section=$media $skip $target @packages &> $done$path/$youri_file"; plog('DEBUG', "running $command"); if (!system($command)) { plog('INFO', "upload succeeded"); } else { # should send a mail or something plog('ERROR', "upload failed ($!), rejecting files in $reject$path/"); make_path("$reject$path"); foreach my $rpm (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) { link("$done$path/${prefix}_$rpm", "$reject$path/${prefix}_$rpm") or plog('ERROR', "ERROR: link of $rpm failed ($!)"); } link("$done$path/$youri_file", "$reject$path/$youri_file"); my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+/; if ($user) { my @pkgs = grep { !/src\.rpm$/ } @{$pkg_tree->{$prefix}{media}{$media}{rpms}}; my $text = join("\n", qq(The upload of the following packages failed:\n), map { "- $_" } @pkgs) . "\n"; my $rpms = join(' ', @pkgs, undef); my $to = get_author_email($user) || "Unknown <$config->{admin}>"; $text .= "\nUpload log available in $config->{http_queue}/rejected/$path/$youri_file\n"; sendmail($to, undef, "Upload failed for $rpms", $text, "Emi the upload bot <$config->{admin}>", 0, $config); } # should delete the files } # delete the files which should have heen either put in queue or rejected unlink(@packages, @duplicate_packages); # unlink the sources rpm unless some non mandatory arch still need to be done my $all_done = 1; if (!defined($pkg_tree->{$prefix}{media}{$media}{arch}{noarch})) { my $arch_list = find { ref($_) eq 'ARRAY' } $config->{arch}, (ref($config->{arch}) eq 'HASH' ? ($config->{arch}{$target}, $config->{arch}{default}) : ()); my @arch_list = $arch_list ? @$arch_list : keys %{$config->{bot}}; # If we are here, mandatory arches are done, no need to check them my $mandatory_arch = get_mandatory_arch($config, $target); foreach my $arch (difference2(\@arch_list, $mandatory_arch)) { next if $pkg_tree->{$prefix}{media}{$media}{arch}{$arch}; $all_done = 0; } } if ($all_done) { foreach (@{$pkg_tree->{$prefix}{srpms}}) { plog('DEBUG', "unlink $todo$path/${prefix}_$_"); unlink("$todo$path/${prefix}_$_"); } } }