1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
package Iurt::Emi;
use base qw(Exporter);
use File::Path qw(make_path);
use Iurt::Config qw(get_author_email get_mandatory_arch get_target_arch);
use Iurt::Mail qw(sendmail);
use Iurt::Queue qw(check_if_all_archs_processed);
use Iurt::Util qw(plog);
use MDK::Common::File qw(touch);
use MDK::Common::Func qw(find);
use MDK::Common::DataStructure qw(difference2);
use strict;
our @EXPORT = qw(
find_prefixes_ready_to_upload
record_uploaded_packages
upload_prefix_in_media
);
sub find_prefixes_ready_to_upload {
my ($config, %pkg_tree) = @_;
# $targets{$target}{$media}{arch_finisher}{$arch}: prefix on which we need to actions to get this arch updated
# $targets{$target}{$media}{to_upload}: list of prefixes to upload
my %targets;
foreach my $prefix (sort keys %pkg_tree) {
my $target = $pkg_tree{$prefix}{target};
my $mandatory_arch = get_mandatory_arch($config, $target);
plog('NOTIFY', "processing $prefix");
my $ok = 1;
foreach my $media (keys %{$pkg_tree{$prefix}{media}}) {
my @wanted_archs = defined($pkg_tree{$prefix}{media}{$media}{done_arch}{noarch}) ? 'noarch' : @$mandatory_arch;
my $path = $pkg_tree{$prefix}{media}{$media}{path};
my %missing;
plog('DEBUG', "... in $path");
if (! $pkg_tree{$prefix}{media}{$media}{rpms}) {
$ok = 0;
last;
}
if ($pkg_tree{$prefix}{media}{$media}{uploaded}) {
plog('INFO', "package already uploaded for mandatory arches, proceeding");
next;
}
foreach my $m (@wanted_archs, 'src') {
$pkg_tree{$prefix}{media}{$media}{excluded_arch}{$m} and next;
my $x = "yes";
if (!$pkg_tree{$prefix}{media}{$media}{done_arch}{$m}) {
$missing{$m} = 1;
$x = "no";
$ok = 0;
}
plog('INFO', " mandatory architecture $m present: $x");
}
unless ($ok) {
plog('INFO', "mandatory arch", join(' ', keys %missing),
"missing for $media, waiting");
next;
}
}
next unless $ok;
#
# All mandatory archs found, mark for upload
#
foreach my $media (keys %{$pkg_tree{$prefix}{media}}) {
$targets{$target}{$media} ||= { 'arch_finisher' => {}, 'to_upload' => [] };
push @{$targets{$target}{$media}{to_upload}}, $prefix;
# We already have found universal finisher in that media, we're fine
next if exists $targets{$target}{$media}{arch_finisher}{noarch};
my %arches_to_upload;
foreach (@{$pkg_tree{$prefix}{media}{$media}{rpms}}) {
next unless /\.([^.]*)\.rpm/;
my $arch = $1;
next if $arch eq 'src';
# If this is a secondary upload, noarch package will be dropped
next if $arch eq 'noarch' && $pkg_tree{$prefix}{media}{$media}{uploaded};
$arches_to_upload{$arch} = 1;
}
if ($arches_to_upload{noarch}) {
# This package is noarch, genhdlist for it will touch all archs
$targets{$target}{$media}{arch_finisher} = { 'noarch' => $prefix };
} else {
my $has_new_arch = scalar(difference2([ keys %arches_to_upload ], [ keys %{$targets{$target}{$media}{arch_finisher}} ]));
if ($has_new_arch) {
# We need this package to cover the new arch
# Set it for all, it may allow getting rid of some
foreach (keys %arches_to_upload) {
$targets{$target}{$media}{arch_finisher}{$_} = $prefix;
}
}
}
}
}
return %targets;
}
sub generate_upload_command {
my ($prefix, $media, $target, $user, $packages, $finish, $youri_file) = @_;
# Most of this function should be in the config...
my $base_command = "/usr/bin/perl -I/usr/share/mga-youri-submit/lib /usr/share/mga-youri-submit/bin/youri-submit --config /etc/youri/submit-upload.conf";
my $all_posts = `$base_command --list posts $target`;
$all_posts =~ s/\n/ /g;
$all_posts =~ s/(^| +)/ --skip-post /g;
$all_posts =~ s/--skip-post *$//;
my $skip = $finish ? "" : $all_posts;
"$base_command --verbose --define user=$user --define prefix=$prefix --define section=$media $skip $target @$packages &> $youri_file";
}
sub upload_prefix_in_media {
my ($config, $pkg_tree, $prefix, $media, $o_finish) = @_;
my $todo = "$config->{queue}/todo";
my $done = "$config->{queue}/done";
my $reject = "$config->{queue}/rejected";
my (@packages, @duplicate_packages);
my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+$/;
my $target = $pkg_tree->{$prefix}{target};
my $youri_file = "$prefix.youri";
if ($pkg_tree->{$prefix}{media}{$media}{uploaded}) {
$youri_file .= "." . time();
}
my $path = $pkg_tree->{$prefix}{media}{$media}{path};
plog('OK', "all mandatory archs done: $prefix");
foreach my $rpm (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) {
my $rpmpath = "$done$path/${prefix}_$rpm";
if ($pkg_tree->{$prefix}{media}{$media}{uploaded}) {
# if already uploaded for mandatory arches, do not try to upload again src or noarch packages
# but still remember these duplicate files for removal
if (my ($type) = $rpm =~ /\.(noarch|src)\.rpm$/) {
plog('DEBUG', "$prefix already uploaded for mandatory arches, not re-uploading $type $rpm");
push @duplicate_packages, $rpmpath;
next;
}
}
push @packages, $rpmpath;
plog('OK', " uploading $rpm in $done/$path");
}
# When a package is not marked as noarch but only has noarch subpackages, there will be nothing
# left to upload when it finishes building on non mandaory arch.
return unless @packages;
my $command = generate_upload_command($prefix, $media, $target, $user, \@packages, $o_finish, "$done$path/$youri_file");
plog('DEBUG', "running $command");
if (!system($command)) {
plog('INFO', "upload succeeded");
} else {
# should send a mail or something
plog('ERROR', "upload failed ($!), rejecting files in $reject$path/");
make_path("$reject$path");
foreach my $rpm (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) {
link("$done$path/${prefix}_$rpm", "$reject$path/${prefix}_$rpm") or plog('ERROR', "ERROR: link of $rpm failed ($!)");
}
link("$done$path/$youri_file", "$reject$path/$youri_file");
my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+/;
if ($user) {
my @pkgs = grep { !/src\.rpm$/ } @{$pkg_tree->{$prefix}{media}{$media}{rpms}};
my $text = join("\n", qq(The upload of the following packages failed:\n), map { "- $_" } @pkgs) . "\n";
my $rpms = join(' ', @pkgs, undef);
my $to = get_author_email($user) || "Unknown <$config->{admin}>";
$text .= "\nUpload log available in $config->{http_queue}/rejected/$path/$youri_file\n";
sendmail($to, undef, "Upload failed for $rpms", $text, "Emi the upload bot <$config->{admin}>", 0, $config);
}
# should delete the files
}
# delete the files which should have heen either put in queue or rejected
unlink(@packages, @duplicate_packages);
# unlink the sources rpm unless some non mandatory arch still need to be done
if (check_if_all_archs_processed($media, $pkg_tree->{$prefix}, $config)) {
foreach (@{$pkg_tree->{$prefix}{srpms}}) {
plog('DEBUG', "unlink $todo$path/${prefix}_$_");
unlink("$todo$path/${prefix}_$_");
}
}
}
sub record_uploaded_packages {
my ($config, $pkg_tree, $prefix, $media) = @_;
my $done = "$config->{queue}/done";
my $path = $pkg_tree->{$prefix}{media}{$media}{path};
my %arches;
foreach my $pkg (@{$pkg_tree->{$prefix}{media}{$media}{rpms}}) {
my ($arch) = $pkg =~ /\.([^.]*)\.rpm$/;
$arches{$arch} = 1 unless $arch eq 'src';
}
# Only keep noarch if it's the only architecture
delete $arches{'noarch'} if 1 < keys %arches;
foreach my $arch (keys %arches) {
touch("$done/$path/${prefix}_$arch.uploaded");
}
touch("$done/$path/$prefix.upload");
}
|
