From 2d4e55f7aa8d46aa2d181e8a352c02be22e79f80 Mon Sep 17 00:00:00 2001 From: Pascal Terjan Date: Tue, 3 Oct 2017 19:13:31 +0100 Subject: Kill upload/rsync in iurt --- NEWS | 1 + iurt | 98 ++++++------------------------------------------------- lib/Iurt/Urpmi.pm | 22 ------------- 3 files changed, 11 insertions(+), 110 deletions(-) diff --git a/NEWS b/NEWS index 20e72ba..32e70fc 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ - iurt: use urpm to parse synthesis (fixes support for xz) - iurt: add fixed_media option to make autobuild faster - iurt: kill a lot of code (and some features) +- iurt: kill upload/rsync code 0.6.29 - iurt: support chroot tarballs with non gz compression diff --git a/iurt b/iurt index 8888c9d..74bfb29 100755 --- a/iurt +++ b/iurt @@ -89,12 +89,12 @@ $run{todo} = []; @params = ( # [ "one letter option", "long name option", "number of args (-X means ´at least X´)", "help text", "function to call", "log info"] # - # no_rsync, config_help and copy_srpm kept for compatibility reasons + # config_help and copy_srpm kept for compatibility reasons # [ "", $program_name, 0, "[--chrooted-urpmi ] [--config foo value] [--warn] [--verbose integer] - [--copy-srpm] [--debug] [--distro] [--no-rsync] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}] + [--copy-srpm] [--debug] [--distro] [--clean user1 user2 user3] [--clean-all] [--shell] [--stop {p|c|i|l|b|a|s}] [--use-system-distrib] [--dir] [--help foo?] [--log filename] [--group] - [--upload [--markrelease] [--source]] [--dir] [--help foo?] [--log filename] [--status] + [--dir] [--help foo?] [--log filename] [--status] [--repository ] [--rpmmacros [...]] {--config-help} @@ -226,28 +226,12 @@ $run{todo} = []; [ "", "rpmmacros", -1, " .. ", "Additional rpm macros to define", sub { $run{rpmmacros} = \@_ }, 'Setting rpm macros' ], - [ "", "upload", [ - ["", "upload", 0, "[options]", - "Upload the rebuild packages", - sub { my ($tmp) = @_; - $tmp->[0] ||= {}; - 1; - }, "Setting upload options"], - [ "m", "markrelease", 0, "", - "Mark SVN directory when uploading the packages", - sub { $run{markrelease} = 1 }, "Adding markrelease repsys option" ], - [ "s", "source", 0, "", - "Upload the source package as wells", - sub { $run{source_upload} = 1 }, "Setting source flag for upload" ], - ], "[options]", - "Upload the rebuild packages", - sub { $run{upload} = 1 }, "Setting the upload flag" ], [ "", "use-old-chroot", 1, "", "Use the given chroot as chroot (usefull for debugging)", sub { ($run{use_old_chroot}) = @_ }, "Using given chroot" ], - [ "", "no_rsync", 0, "", - "Do not send build log to the distant rsync server", - sub { $run{no_rsync} = 1 }, "Setting the no rsync warn flag" ], + [ "", "no_rsync", 0, "", + "Do not send build log to the distant rsync server", + sub { $run{no_rsync} = 1 }, "Setting the no rsync flag" ], [ "", "delete-on-success", 0, "", "Don't keep generated packages and their logs", sub { $run{delete_on_success} = 1 }, "Setting the delete on success flag" ], @@ -496,10 +480,6 @@ my %config_usage = ( desc => 'Prefix of the repositories', default => '' }, - rsync_to => { - desc => 'Server where the result of the builds should be rsynced (name@server:path format)', - default => '' - }, sendmail => { desc => 'If the bot will send mail reports regarding build', default => 0 @@ -508,10 +488,6 @@ my %config_usage = ( desc => 'Table of supported architecture', default => ['i586', 'x86_64'] }, - upload => { - desc => 'Where to copy build packages', - default => "$HOME/uploads/" - }, vendor => { desc => 'Name of the packages vendor', default => 'Mageia.Org' @@ -555,8 +531,6 @@ if ($config->{env}) { } } -$config->{upload} .= "$run{distro}/$run{media}"; - if ($run{icecream}) { push @{$config->{basesystem_packages}}, 'icecream'; } @@ -605,10 +579,10 @@ if ($run{resume}) { empty_status($local_spool, \%run); } -my (%srpm_version, @wrong_rpm, %provides, %pack_provide, $to_compile, %maint); +my (%srpm_version, %provides, %pack_provide, $to_compile, %maint); $to_compile = @{$run{todo}}; $to_compile += check_media(\%run, $cache, $config, \%srpm_version, - \@wrong_rpm, \%provides, \%pack_provide, \%maint) if $run{media}; + \%provides, \%pack_provide, \%maint) if $run{media}; $to_compile += search_packages(1, $cache, \%provides, \%run, \%maint, \%srpm_version, @{$run{extra_dir}}) if $run{extra}; @@ -711,15 +685,6 @@ if (!$run{stop}) { close STDIN; } -# perform some cleaning before running to have some more space, rsync to -# the server too in case previous iurt crashed - -if ($config->{rsync_to} && !$run{no_rsync}) { - # remove some old and very big log files not to saturate the server - system(qq(find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;)); - system('rsync', '--delete', '-alHPe', 'ssh -x', "$local_spool/log/", "$config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/"); -} - # The next loop should be moved in a module someday # FIXME: (tv) kill this dead code or use it!! @@ -933,7 +898,6 @@ retry: unlink "$local_spool/$srpm"; } } - process_queue($config, \%run, \@wrong_rpm, 1); } return $srpm; } @@ -1033,21 +997,6 @@ do { if (!$run{debug} && !$run{use_old_chroot}) { clean_chroot($chroot_tmp, \%run, $config); } -plog("reprocess generated packages queue"); -process_queue($config, \%run, \@wrong_rpm); - -plog('FAIL', "ERROR: RPM with a wrong SRPM name") if @wrong_rpm; -if (@wrong_rpm && open my $file, ">$local_spool/log/wrong_srpm_names.log") { - foreach (@wrong_rpm) { - print $file "$_->[1] -> $_->[0]\n"; - } -} - -if ($config->{rsync_to} && !$run{no_rsync}) { - # remove some old and very big log files not to saturate the server - system(qq(find $local_spool/log/ -name "*.log" \\( -size +$config->{log_size_limit} -or -mtime +$config->{log_size_date} \\) -exec rm -f {} \\;)); - system('rsync', '--delete', '-alHPe', 'ssh -x', "$local_spool/log/", "$config->{rsync_to}/$run{distro_tag}/$run{my_arch}/$run{media}/log/"); -} unlink "$run{pidfile_home}/$run{pidfile}" if $run{pidfile}; @@ -1058,33 +1007,6 @@ exit(); # # -sub process_queue { - my ($config, $run, $wrong_rpm, $o_quiet) = @_; - return if !$run->{upload} && $o_quiet; - my $dir = "$config->{local_upload}/iurt/$run->{distro_tag}/$run->{my_arch}/$run->{media}/"; - opendir my $rpmdir, $dir or return; - my $urpmi = $run->{urpmi}; - foreach my $rpm (readdir $rpmdir) { - my ($rarch, $srpm) = $urpmi->update_srpm($dir, $rpm, $wrong_rpm); - $rarch or next; - plog($rpm); - next if !$run->{upload}; - # recheck if the package has not been uploaded in the meantime - my $rpms_dir = "$config->{repository}/$run->{distro}/$run->{my_arch}/media/$run->{media}/"; - if (! -f "$rpms_dir/$rpm") { - my $err = system('/usr/bin/scp', "$dir/$rpm", $config->{upload} . "/$config->{extra_subdir}/RPMS/"); - # try to keep the opportunity to prevent disk full - if ($err) { - plog("ERROR: process_queue: cannot copy $dir/$rpm to ", $config->{upload}, "/$config->{extra_subdir}/RPMS/ ($!)"); - next; - } - } - unlink "$dir/$rpm"; - $cache->{queue}{$srpm} = 1; - } - closedir $rpmdir; -} - sub check_version { my ($run, $srpm, $srpm_version) = @_; my ($srpm_name) = $srpm =~ /(.*)-[^-]+-[^-]+\.src\.rpm/; @@ -1183,7 +1105,7 @@ sub check_pid { } sub check_media { - my ($run, $cache, $config, $srpm_version, $wrong_rpm, $provides, $pack_provide, $maint) = @_; + my ($run, $cache, $config, $srpm_version, $provides, $pack_provide, $maint) = @_; # We could rely only on parsing the synthesis, hoping that they are correct, however this scan is very fast, so... if (!$run->{build_all}) { foreach my $subdir (@{$config->{all_media}{$run->{media}}}) { @@ -1192,7 +1114,7 @@ sub check_media { opendir my $rpmdir, $rpms_dir or die "Could not open $rpms_dir: $!"; my $urpmi = $run->{urpmi}; foreach my $rpm (readdir $rpmdir) { - my ($rarch, $srpm) = $urpmi->update_srpm($rpms_dir, $rpm, $wrong_rpm); + my ($rarch, $srpm) = $urpmi->update_srpm($rpms_dir, $rpm); $rarch or next; $cache->{queue}{$srpm} = 1; $run{status}{$srpm} = 'ok'; diff --git a/lib/Iurt/Urpmi.pm b/lib/Iurt/Urpmi.pm index baa6b92..41186a8 100644 --- a/lib/Iurt/Urpmi.pm +++ b/lib/Iurt/Urpmi.pm @@ -524,28 +524,6 @@ sub clean_urpmi_process { } } -sub update_srpm { - my ($self, $dir, $rpm, $wrong_rpm) = @_; - my $run = $self->{run}; - my ($arch) = $rpm =~ /([^\.]+)\.rpm$/ or return 0; - if (!$srpm) { - my $hdr = RPM4::Header->new("$dir/$rpm"); - $hdr or return 0; - $srpm = $hdr->queryformat('%{SOURCERPM}'); - } - $srpm = fix_srpm_name($srpm, $rpm, $wrong_rpm); - $arch, $srpm; -} - -sub fix_srpm_name { - my ($srpm, $rpm, $wrong_rpm) = @_; - my $old_srpm = $srpm; - if ($srpm =~ s/^lib64/lib/) { - push @$wrong_rpm, [ $old_srpm, $rpm ] if ref $wrong_rpm; - } - $srpm; -} - # return ("exit_code", srpm, spec) sub recreate_srpm { my ($_self, $run, $config, $chroot_tmp, $dir, $srpm, $luser, $b_retry) = @_; -- cgit v1.2.1