package Iurt::Emi; use base qw(Exporter); use File::Path qw(make_path); use Iurt::Config qw(get_author_email get_mandatory_arch get_target_arch); use Iurt::Mail qw(sendmail); use Iurt::Queue qw(check_if_all_archs_processed); use Iurt::Util qw(plog); use MDK::Common::File qw(touch); use MDK::Common::Func qw(find); use MDK::Common::DataStructure qw(difference2); use strict; our @EXPORT = qw( find_prefixes_ready_to_upload record_uploaded_packages upload_prefix_in_media ); sub find_prefixes_ready_to_upload { my ($config, %pkg_tree) = @_; # $targets{$target}{$media}{arch_finisher}{$arch}: prefix on which we need to actions to get this arch updated # $targets{$target}{$media}{to_upload}: list of prefixes to upload my %targets; foreach my $prefix (sort keys %pkg_tree) { my $target = $pkg_tree{$prefix}{target}; my $mandatory_arch = get_mandatory_arch($config, $target); plog('NOTIFY', "processing $prefix"); my $ok = 1; foreach my $media (keys %{$pkg_tree{$prefix}{media}}) { my @wanted_archs = defined($pkg_tree{$prefix}{media}{$media}{done_arch}{noarch}) ? 'noarch' : @$mandatory_arch; my $path = $pkg_tree{$prefix}{media}{$media}{path}; my %missing; plog('DEBUG', "... in $path"); if (! $pkg_tree{$prefix}{media}{$media}{rpms}) { $ok = 0; last; } if ($pkg_tree{$prefix}{media}{$media}{uploaded}) { plog('INFO', "package already uploaded for mandatory arches, proceeding"); next; } foreach my $m (@wanted_archs, 'src') { $pkg_tree{$prefix}{media}{$media}{excluded_arch}{$m} and next; my $x = "yes"; if (!$pkg_tree{$prefix}{media}{$media}{done_arch}{$m}) { $missing{$m} = 1; $x = "no"; $ok = 0; } plog('INFO', " mandatory architecture $m present: $x"); } unless ($ok) { plog('INFO', "mandatory arch", join(' ', keys %missing), "missing for $media, waiting"); next; } } next unless $ok; # # All mandatory archs found, mark for upload # foreach my $media (keys %{$pkg_tree{$prefix}{media}}) { $targets{$target}{$media} ||= { 'arch_finisher' => {}, 'to_upload' => [] }; push @{$targets{$target}{$media}{to_upload}}, $prefix; # We already have found universal finisher in that media, we're fine next if exists $targets{$target}{$media}{arch_finisher}{noarch}; my %arches_to_upload; foreach (@{$pkg_tree{$prefix}{media}{$media}{rpms}}) { next unless /\.([^.]*)\.rpm/; my $arch = $1; next if $arch eq 'src'; # If this is a secondary upload, noarch package will be dropped next if $arch eq 'noarch' && $pkg_tree{$prefix}{media}{$media}{uploaded}; $arches_to_upload{$arch} = 1; } if ($arches_to_upload{noarch}) { # This package is noarch, genhdlist for it will touch all archs $targets{$target}{$media}{arch_finisher} = { 'noarch' => $prefix }; } else { my $has_new_arch = scalar(difference2([ keys %arches_to_upload ], [ keys %{$targets{$target}{$media}{arch_finisher}} ])); if ($has_new_arch) { # We need this package to cover the new arch # Set it for all, it may allow getting rid of some foreach (keys %arches_to_upload) { $targets{$target}{$media}{arch_finisher}{$_} = $prefix; } } } } } return %targets; } sub generate_upload_command { my ($prefix, $media, $target, $user, $packages, $finish, $youri_file) = @_; # Most of this function should be in the config... my $base_command = "/usr/bin/perl -I/usr/share/mga-youri-submit/lib /usr/share/mga-youri-submit/bin/youri-submit --config /etc/youri/submit-upload.conf"; my $all_posts = `$base_command --list posts $target`; $all_posts =~ s/\n/ /g; $all_posts =~ s/(^| +)/ --skip-post /g; $all_posts =~ s/--skip-post *$//; my $skip = $finish ? "" : $all_posts; "$base_command --verbose --define user=$user --define prefix=$prefix --define section=$media $skip $target @$packages &> $youri_file"; } sub upload_prefix_in_media { my ($config, $pkg_tree, $prefix, $media, $o_finish) = @_; my $todo = "$config->{queue}/todo"; my $done = "$config->{queue}/done"; my $reject = "$config->{queue}/rejected"; my (@packages, @duplicate_packages); my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+$/; my $target = $pkg_tree->{$prefix}{target}; my $youri_file = "$prefix.youri"; if ($pkg_tree->{$prefix}{media}{$media}{uploaded}) { $youri_file .= "." . time(); } my $path = $pkg_tree->{$prefix}{media}{$media}{path}; plog('OK', "all mandatory archs done: $prefix"); foreach my $rpm (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) { my $rpmpath = "$done$path/${prefix}_$rpm"; if ($pkg_tree->{$prefix}{media}{$media}{uploaded}) { # if already uploaded for mandatory arches, do not try to upload again src or noarch packages # but still remember these duplicate files for removal if (my ($type) = $rpm =~ /\.(noarch|src)\.rpm$/) { plog('DEBUG', "$prefix already uploaded for mandatory arches, not re-uploading $type $rpm"); push @duplicate_packages, $rpmpath; next; } } push @packages, $rpmpath; plog('OK', " uploading $rpm in $done/$path"); } # When a package is not marked as noarch but only has noarch subpackages, there will be nothing # left to upload when it finishes building on non mandaory arch. return unless @packages; my $command = generate_upload_command($prefix, $media, $target, $user, \@packages, $o_finish, "$done$path/$youri_file"); plog('DEBUG', "running $command"); if (!system($command)) { plog('INFO', "upload succeeded"); } else { # should send a mail or something plog('ERROR', "upload failed ($!), rejecting files in $reject$path/"); make_path("$reject$path"); foreach my $rpm (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) { link("$done$path/${prefix}_$rpm", "$reject$path/${prefix}_$rpm") or plog('ERROR', "ERROR: link of $rpm failed ($!)"); } link("$done$path/$youri_file", "$reject$path/$youri_file"); my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+/; if ($user) { my @pkgs = grep { !/src\.rpm$/ } @{$pkg_tree->{$prefix}{media}{$media}{rpms}}; my $text = join("\n", qq(The upload of the following packages failed:\n), map { "- $_" } @pkgs) . "\n"; my $rpms = join(' ', @pkgs, undef); my $to = get_author_email($user) || "Unknown <$config->{admin}>"; $text .= "\nUpload log available in $config->{http_queue}/rejected/$path/$youri_file\n"; sendmail($to, undef, "Upload failed for $rpms", $text, "Emi the upload bot <$config->{admin}>", 0, $config); } # should delete the files } # delete the files which should have heen either put in queue or rejected unlink(@packages, @duplicate_packages); # unlink the sources rpm unless some non mandatory arch still need to be done if (check_if_all_archs_processed($media, $pkg_tree->{$prefix}, $config)) { foreach (@{$pkg_tree->{$prefix}{srpms}}) { plog('DEBUG', "unlink $todo$path/${prefix}_$_"); unlink("$todo$path/${prefix}_$_"); } } } sub record_uploaded_packages { my ($config, $pkg_tree, $prefix, $media) = @_; my $done = "$config->{queue}/done"; my $path = $pkg_tree->{$prefix}{media}{$media}{path}; my %arches; foreach my $pkg (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) { my ($arch) = $pkg =~ /\.([^.]*)\.rpm$/; $arches{$arch} = 1 unless $arch eq 'src'; } # Only keep noarch if it's the only architecture delete $arches{'noarch'} if 1 < keys %arches; foreach my $arch (keys %arches) { touch("$done/$path/${prefix}_$arch.uploaded"); } touch("$done/$path/$prefix.upload"); }