package MGA::Advisories; use warnings; use strict; use YAML qw(LoadFile DumpFile Load); use Template; use DateTime; use Email::Sender::Simple qw(try_to_sendmail); use Email::Simple; use Email::Simple::Creator; use HTTP::Request; use LWP::UserAgent; use File::Basename; use XMLRPC::Lite; use Term::ReadKey; #use Data::Dump qw(dd); our $config_file = '/usr/share/mga-advisories/config'; our $config = LoadFile($ENV{MGAADV_CONF} || $config_file); our $home_config_file = $ENV{HOME} . '/.mga-advisories/mga-advisories.conf'; my $custom_config_file = -f $home_config_file ? $home_config_file : '/etc/mga-advisories.conf'; my $custom_config = LoadFile($custom_config_file); foreach my $k (keys %$custom_config) { $config->{$k} = $custom_config->{$k}; } my %basename = ( CVE => sub { $_[0] }, ID => sub { $_[0] }, rel => sub { $_[0] }, src => sub { 'src_' . $_[0] }, ); my %tools = ( pkgname => sub { $_[0] =~ m/(.+)-[^-]+-[^-]+/; $1; }, ); my %bz = ( proxy => undef, token => undef ); my @report_logs; sub report_log { push @report_logs, @_; } sub report_exit { report_log($_[0]); send_report({ error => $_[0] }); exit 1; } sub status_file { $config->{status_dir} . '/' . $_[0]; } sub save_status { my ($advdb, $adv) = @_; return if $advdb->{advisories}{$adv}{no_save_status}; my $statusfile = status_file($adv); DumpFile($statusfile, $advdb->{advisories}{$adv}{status}); } sub init_bz { return 1 if ($bz{proxy}); return 0 if (!$config->{bugzilla_url}); $bz{proxy} = XMLRPC::Lite->proxy( $config->{bugzilla_url} . "xmlrpc.cgi" ); if ($config->{bugzilla_tokenfile} && -f glob($config->{bugzilla_tokenfile})) { if (open(my $fh, '<', glob($config->{bugzilla_tokenfile}))) { while (my $row = <$fh>) { chomp($bz{token} = $row); last; } close $fh; } } #$bz{proxy}->import(+trace => 'debug'); return 1; } sub call_bz { my ($method, @args) = @_; return 0 if (!init_bz()); $args[0]->{token} = $bz{token} if ($bz{token}); my $soapresult = $bz{proxy}->call($method, @args); if ($soapresult->fault) { if ($soapresult->faultcode == 410) { # We need to login if (login_bz()) { $args[0]->{token} = $bz{token}; # Try the call again now we're logged in. $soapresult = $bz{proxy}->call($method, @args); return $soapresult->result unless $soapresult->fault; } } my ($package, $filename, $line) = caller; report_log( $soapresult->faultcode . ' ' . $soapresult->faultstring . " in SOAP call near $filename line $line." ); return 0; } return $soapresult->result; } sub login_bz { my ($login, $password); if ($config->{bugzilla_login} && $config->{bugzilla_password}) { $login = $config->{bugzilla_login}; $password = $config->{bugzilla_password}; } elsif ($config->{mode} eq 'qa') { print "Please enter your bugzilla user name (email address): "; chomp($login = ); print "Please now enter your bugzilla password: "; ReadMode('noecho'); chomp($password = ); ReadMode(0); # back to normal print "\n"; } if ($login && $password) { if ( $password =~ /^file:\/\// ) { if (open(my $fh, '<:encoding(UTF-8)', substr $password, 7)) { while (my $row = <$fh>) { chomp($password = $row); last; } close $fh; } else { print STDERR "Warning: Cannot open bugzilla password file\n"; return 0; } } my $soapresult = $bz{proxy}->call( 'User.login', { login => $login, password => $password, remember => 1 } ); if ($soapresult->result && $soapresult->result->{token}) { $bz{token} = $soapresult->result->{token}; if ($config->{bugzilla_tokenfile}) { if (open(my $fh, '>', glob($config->{bugzilla_tokenfile}))) { print $fh $bz{token}; close $fh; chmod 0600, glob($config->{bugzilla_tokenfile}); } } } return 1 unless $soapresult->fault; } return 0; } sub get_advisories_from_dir { my %advisories; foreach my $advfile (glob "$config->{advisories_dir}/*.adv") { my $adv = LoadFile($advfile); if (!$adv->{ID}) { next unless $config->{mode} eq 'qa'; $adv->{ref} = basename($advfile, ".adv"); $adv->{ID} = next_id('TODO', keys %advisories); $adv->{no_save_status} = 1; } report_exit("Duplicate advisory $adv->{ID}") if $advisories{$adv->{ID}}; report_exit("Unknown type $adv->{type}") unless $config->{advisory_types}{$adv->{type}}; $advisories{$adv->{ID}} = $adv; my $statusfile = status_file($adv->{ID}); $adv->{status} = -f $statusfile ? LoadFile($statusfile) : {}; } return \%advisories; } sub next_id { my $prefix = shift; my $year = DateTime->now->year; my @used_ids = map { m/^$prefix-$year-(\d+)$/ ? int $1 : () } @_; my $newid = (0, sort { $a <=> $b } @used_ids)[-1] + 1; return sprintf("%s-%s-%.4d", $prefix, $year, $newid); } sub assign_id { my ($advname) = @_; chdir($config->{advisories_dir}); my $advfile = "$advname.adv"; $advfile =~ s/\.adv\.adv$/.adv/; if ( ! -f $advfile ) { print STDERR "Cannot find advisory file '$advname'\n"; return; } my $exitstatus = system("svn status -q $advfile | grep -q ^M"); if ( 0 == $exitstatus ) { print STDERR "$advname appears to be modified. Please commit it first.\n"; return; } my $adv = LoadFile($advfile); if ($adv->{ID}) { print STDERR "$advname already has an ID assigned: $adv->{ID}\n"; return; } my $type = $config->{advisory_types}{$adv->{type}}{prefix}; if (!$type) { print STDERR "Unknown type $adv->{type}\n"; return; } # Turn on autoflush $|++; if (!init_bz()) { print STDERR "Warning: Cannot check bugzilla. Please double check manually\n"; } else { # Advisories are not always just [0-9]+.adv, but are # sometimes [0-9]+.mga3.adv etc. so extract the real bug number (my $advbugnum) = $advname =~ m/^([0-9]+)/g; if (scalar($adv->{references}) < 1) { print STDERR "No reference links found in advisory\n"; return; } (my $bugnum) = $adv->{references}[0] =~ m/$config->{bugzilla_url}.*=([0-9]+)$/g; if (!defined $bugnum) { print STDERR "First reference does not appear to be a Mageia bug link\n"; return; } if ($advbugnum ne $bugnum) { print STDERR "First reference is not the corresponding Mageia advisory bug ($advbugnum != $bugnum)\n"; return; } if (my $result = call_bz('Bug.get', {ids => [$bugnum]})) { my $failed = 0; my $bug = $result->{bugs}->[0]; print "Found Bug: " . $bug->{summary} . "\n " . $config->{bugzilla_url} . "$bugnum\n"; my ($buffer, $msg); print $msg = sprintf("%-40s", "Checking for QA validation keyword… "); $buffer = $msg; my $found_keyword = 0; if (scalar($bug->{keywords}) > 0) { foreach my $keyword (@{$bug->{keywords}}) { if ('validated_update' eq $keyword) { $found_keyword = 1; last; } } } if ($found_keyword) { print $msg = "✔\n"; } else { print $msg = "✘\n"; $failed = 1; } $buffer .= $msg; my $depsfailed = 0; print $msg = sprintf("%-40s", "Checking dependent bugs… "); $buffer .= $msg; my $depends = $bug->{depends_on}; if (scalar(@$depends) < 1) { print $msg = "✔ (None found)\n"; $buffer .= $msg; } else { my $first = 1; foreach my $dependent_bug_num (@$depends) { if (!$first) { print $msg = ', '; $buffer .= $msg; } $first = 0; if ($result = call_bz('Bug.get', {ids => [$dependent_bug_num]})) { my $blocking_bug = $result->{bugs}->[0]; if ($blocking_bug->{is_open}) { print $msg = "✘ $dependent_bug_num"; $depsfailed = 1; } else { print $msg = "✔ $dependent_bug_num"; } } else { print "? $dependent_bug_num\n"; print STDERR "Error: There was a problem communicating with bugzilla for bug $dependent_bug_num\n"; return; } $buffer .= $msg; } print $msg = "\n"; $buffer .= $msg; } if (!$failed && $depsfailed) { ReadMode 4; # Turn off controls keys print $msg = "Dependent bug! Publish anyway? [y/N]: "; $buffer .= $msg; my $key = 'x'; while ( $key ne "\n" && $key ne "y" && $key ne "Y" && $key ne "n" && $key ne "N" ) { $key = ReadKey(0); } ReadMode 0; # Reset tty mode before exiting if ( $key eq "\n" || $key eq "n" || $key eq "N" ) { $failed = 1; print $msg = " ✘\n"; } else { print $msg = " ✔\n"; } $buffer .= $msg; } print $msg = sprintf("%-40s", "Checking SRPMs… "); $buffer .= $msg; my $ua = LWP::UserAgent->new; $ua->max_redirect(0); foreach my $rel (keys %{$adv->{src}}) { foreach my $media (keys %{$adv->{src}{$rel}}) { foreach my $srpm (@{$adv->{src}{$rel}{$media}}) { my $req = HTTP::Request->new(GET => "http://repository.mageia.org/qa/checksrpm/update/$rel/$media/$srpm"); my $resp = $ua->request($req); if ($resp->code eq 302 && $resp->header('Location') =~ /\/qa\/checksrpm\/found$/) { print $msg = "✔ "; } else { print $msg = "✘ ($rel/$media/$srpm) "; $failed = 1; } $buffer .= $msg; } } } print $msg = "\n"; $buffer .= $msg; if ($failed) { print STDERR "Error: Cross check failed.\n"; if ($found_keyword) { ReadMode 4; # Turn off controls keys print "Post failure message to Bugzilla and reset 'validated_update' keyword (requires login)? [y/N]: "; my $key = 'x'; while ( $key ne "\n" && $key ne "y" && $key ne "Y" && $key ne "n" && $key ne "N" ) { $key = ReadKey(0); } ReadMode 0; # Reset tty mode before exiting if ( $key eq "\n" || $key eq "n" || $key eq "N" ) { print " ✘\n"; } else { print " ✔\n"; if (call_bz('Bug.update', { ids => [$bugnum], comment => { body => "Update ID assignment failed\n\n$buffer\n\n'validated_update' keyword reset." }, keywords => { set => grep { $_ ne 'validated_update' } @{$bug->{keywords}} } })) { print "Successfully posted to Bugzilla\n"; } else { print "Failed to post to Bugzilla\n"; } } } return; } } else { print STDERR "Warning: Cannot check bugzilla. Please double check manually\n"; } } # TODO: Check SRPMs really exist in the media printf "%-40s", "Assigning ID to advisory $advname… "; $adv->{ID} = next_id($type, keys %{get_advisories_from_dir()}); open(my $fh, '>>', $advfile) or die "Error opening $advfile"; print $fh "ID: $adv->{ID}\n"; close $fh; print "✔ $adv->{ID}\n"; ReadMode 4; # Turn off controls keys print "Do you want to publish now? [Y/n]: "; my $key = 'x'; while ( $key ne "\n" && $key ne "y" && $key ne "Y" && $key ne "n" && $key ne "N" ) { $key = ReadKey(0); } ReadMode 0; # Reset tty mode before exiting if ( $key ne "\n" && $key ne "y" && $key ne "Y" ) { print "$key\n"; return; } print " ✔\n"; printf "%-40s", "Publishing advisory $adv->{ID}… "; my $message = $adv->{ID}; my @pkgs; foreach my $rel (keys %{$adv->{src}}) { foreach my $media (keys %{$adv->{src}{$rel}}) { foreach my $srpm (@{$adv->{src}{$rel}{$media}}) { push @pkgs, $srpm; } } } $message .= ': '.join(', ', @pkgs); $exitstatus = system('svn', 'commit', '-q', '-m', $message, $advfile); if (0 == $exitstatus) { print "✔\n"; } else { print "✘\n"; } } sub assign_ids { chdir($config->{advisories_dir}); # Ensure users local repo is up to date my $exitstatus = system("svn up"); if ( 0 != $exitstatus ) { print STDERR "Subversion update appears to have failed.\n"; return; } my %advdb; $advdb{advisories} = get_advisories_from_dir(); sort_advisories(\%advdb); output_pages(); # We will have exited by now in the event of e.g. a Yaml or processing error foreach my $advfile (glob "$config->{advisories_dir}/*.adv") { my $adv = LoadFile($advfile); next if ($adv->{ID}); assign_id(basename($advfile, ".adv")); print "\n"; } } sub advdb_dumpfile { $config->{advdb_dumpfile} || $ENV{HOME} . '/.mga-advisories/advisories.yaml'; } sub get_advisories_from_dump { my $advfile = advdb_dumpfile; return -f $advfile ? LoadFile($advfile) : {}; } sub get_advisories { return $config->{mode} eq 'dump' ? get_advisories_from_dump : get_advisories_from_dir; } sub download_advisories { my $oldadvisories = get_advisories_from_dump; my $ua = LWP::UserAgent->new; my $resp = $ua->get($config->{dump_url}); die "Error loading $config->{dump_url}" unless $resp->is_success; my $newadvisories = Load($resp->decoded_content); my @newadv = grep { ! $oldadvisories->{$_} } keys %$newadvisories; if (@newadv) { my %n; my @v = @{$newadvisories}{@newadv}; @n{@newadv} = @v; print "New advisories have been downloaded :\n"; listadv({advisories => \%n}); } else { print "No new advisories available\n"; } if (!-d dirname(advdb_dumpfile)) { mkdir dirname(advdb_dumpfile) || die "Error creating directory " . dirname(advdb_dumpfile); } open(my $fh, '>', advdb_dumpfile) || die "Could not open " . advdb_dumpfile; print $fh $resp->decoded_content; close $fh; } sub move_packages { my ($advdb) = @_; return unless $config->{mode} eq 'site'; return unless $config->{move_pkg_cmd}; my @cmd = ( $config->{move_pkg_cmd}, '--sync', '--no-confirm' ); foreach my $adv (keys %{$advdb->{advisories}}) { next if $advdb->{advisories}{$adv}{status}{moved}; foreach my $rel (keys %{$advdb->{advisories}{$adv}{src}}) { foreach my $media (keys %{$advdb->{advisories}{$adv}{src}{$rel}}) { foreach my $srpm (@{$advdb->{advisories}{$adv}{src}{$rel}{$media}}) { report_log("Moving package $srpm for $adv"); push(@cmd, "$rel/$media/$srpm.src.rpm"); } } } } return if scalar(@cmd) < 4; report_exit("Could not move packages") if (0 != system(@cmd)); foreach my $adv (keys %{$advdb->{advisories}}) { next if $advdb->{advisories}{$adv}{status}{moved}; $advdb->{advisories}{$adv}{status}{moved} = time(); save_status($advdb, $adv); } } sub publish_advisories { my ($advdb) = @_; foreach my $adv (sort keys %{$advdb->{advisories}}) { next if $advdb->{advisories}{$adv}{status}{published}; $advdb->{advisories}{$adv}{status}{published} = $advdb->{advisories}{$adv}{pubtime} || time(); save_status($advdb, $adv); } } sub adv_sort { my $advdb = shift; sort { my $now = time; my $pa = $advdb->{advisories}{$a}{status}{published} || $now; my $pb = $advdb->{advisories}{$b}{status}{published} || $now; return $pa == $pb ? $b cmp $a : $pb cmp $pa; } @_; } sub sort_advisories { my ($advdb) = @_; foreach my $adv (keys %{$advdb->{advisories}}) { push @{$advdb->{by_type}{$advdb->{advisories}{$adv}{type}}}, $adv; foreach my $cve (@{$advdb->{advisories}{$adv}{CVE}}) { push @{$advdb->{by_cve}{$cve}}, $adv; } foreach my $rel (keys %{$advdb->{advisories}{$adv}{src}}) { push @{$advdb->{by_rel}{$rel}}, $adv; foreach my $media (keys %{$advdb->{advisories}{$adv}{src}{$rel}}) { push @{$advdb->{by_media}{$media}}, $adv; foreach my $srpm (@{$advdb->{advisories}{$adv}{src}{$rel}{$media}}) { my $pkgname = $tools{pkgname}->($srpm); if ($pkgname) { push @{$advdb->{by_src}{$pkgname}}, $adv unless grep { $_ eq $adv } @{$advdb->{by_src}{$pkgname}}; } else { print STDERR "Warning: Invalid SRPM '$srpm' for advisory '$adv'\n"; } } } } } foreach my $by ('by_type', 'by_cve', 'by_rel', 'by_media', 'by_src') { foreach my $k (keys %{$advdb->{$by}}) { $advdb->{$by}{$k} = [ adv_sort($advdb, @{$advdb->{$by}{$k}}) ]; } } $advdb->{sorted} = [ adv_sort($advdb, keys %{$advdb->{advisories}}) ]; } sub process_template { my ($template, $src, $vars, $dest, $ext) = @_; foreach my $extension ($ext ? $ext : @{$config->{output_format}}) { next unless -f "$config->{tmpl_dir}/$src.$extension"; $template->process("$src.$extension", $vars, ref $dest ? $dest : "$dest.$extension", binmode => ':utf8') || die $template->error, "\n"; } } sub output_pages { my ($advdb) = @_; my $template = Template->new( ENCODING => 'utf8', INCLUDE_PATH => $config->{tmpl_dir}, OUTPUT_PATH => $config->{out_dir}, ); foreach my $adv (keys %{$advdb->{advisories}}) { my $vars = { config => $config, advisory => $adv, advdb => $advdb, basename => \%basename, tools => \%tools, }; process_template($template, 'advisory', $vars, $basename{ID}->($adv)); } foreach my $by (['rel', 'by_rel'], ['CVE', 'by_cve'], ['src', 'by_src']) { foreach my $r (keys %{$advdb->{$by->[1]}}) { my $vars = { config => $config, $by->[0] => $r, advdb => $advdb, basename => \%basename, tools => \%tools, }; process_template($template, $by->[1], $vars, $basename{$by->[0]}->($r)); } } my $vars = { config => $config, advdb => $advdb, basename => \%basename, tools => \%tools, }; process_template($template, 'index', $vars, 'index'); process_template($template, 'advisories', $vars, 'advisories'); process_template($template, 'infos', $vars, 'infos'); process_template($template, 'CVE', $vars, 'CVE'); } sub send_adv_mail { my ($advdb) = @_; return unless $config->{send_adv_mail} eq 'yes'; return unless $config->{mode} eq 'site'; my $template = Template->new( ENCODING => 'utf8', INCLUDE_PATH => $config->{tmpl_dir}, ); foreach my $adv (sort keys %{$advdb->{advisories}}) { next if $advdb->{advisories}{$adv}{no_mail}; next if $advdb->{advisories}{$adv}{no_save_status}; next if $advdb->{advisories}{$adv}{status}{mail_sent}; my $mailcontent; my $vars = { config => $config, advisory => $adv, advdb => $advdb, basename => \%basename, tools => \%tools, }; process_template($template, 'advisory', $vars, \$mailcontent, 'txt'); my $email = Email::Simple->create( header => [ To => $config->{adv_mail_to}, From => $config->{adv_mail_from}, Subject => "$adv: " . $advdb->{advisories}{$adv}{subject}, ], body => $mailcontent ); if (try_to_sendmail($email)) { report_log("Advisory mail for $adv sent"); $advdb->{advisories}{$adv}{status}{mail_sent} = time(); save_status($advdb, $adv); # Attempt to make the mail delivery "in order" sleep 1; } else { report_log("Error sending advisory mail $adv"); } } } sub close_bugs { my ($advdb) = @_; return unless $config->{mode} eq 'site'; if (!init_bz()) { report_log("Cannot close advisory bugs. Could not initialise bugzilla"); return; } foreach my $adv (sort keys %{$advdb->{advisories}}) { next if $advdb->{advisories}{$adv}{status}{bug_closed}; my $bugnum = 0; if (scalar($advdb->{advisories}{$adv}{references}) > 0) { ($bugnum) = $advdb->{advisories}{$adv}{references}[0] =~ m/$config->{bugzilla_url}.*=([0-9]+)$/g; } if (!$bugnum) { report_log("Could not extract bug number for $adv"); next; } my $advurl = $config->{site_url} . '/' . $adv . '.html'; my $comment = "An update for this issue has been pushed to Mageia Updates repository.\n\n$advurl"; if (my $result = call_bz('Bug.update', { ids => [$bugnum], status => 'RESOLVED', resolution => 'FIXED', comment => { body => $comment } })) { report_log("Bug $bugnum closed for $adv"); $advdb->{advisories}{$adv}{status}{bug_closed} = time(); save_status($advdb, $adv); } else { report_log("Could not close bug $bugnum for $adv"); } } } sub send_report { my ($advdb) = @_; return unless @report_logs; my $template = Template->new( ENCODING => 'utf8', INCLUDE_PATH => $config->{tmpl_dir}, ); my $reportcontent; my $vars = { config => $config, advdb => $advdb, report_logs => \@report_logs, }; process_template($template, 'report', $vars, \$reportcontent, 'txt'); if ($config->{send_report_mail} eq 'yes' && $config->{mode} eq 'site') { my $email = Email::Simple->create( header => [ To => $config->{report_mail_to}, From => $config->{report_mail_from}, Subject => $advdb->{error} ? 'Advisories Error' : 'Advisories Update', ], body => $reportcontent ); try_to_sendmail($email); } else { print { $advdb->{error} ? *STDERR : *STDOUT } $reportcontent; } } sub dumpdb { my ($advdb) = @_; DumpFile($config->{out_dir} . '/advisories.yaml', $advdb->{advisories}); } sub newadv { my ($type, $bugnum) = @_; my $file = $config->{advisories_dir} . '/' . $bugnum . '.adv'; if (-f $file) { print STDERR "File $file already exists\n"; return undef; } my $template = Template->new( INCLUDE_PATH => $config->{tmpl_dir}, OUTPUT_PATH => $config->{advisories_dir}, ENCODING => 'utf8', ); my $vars = { type => $type, bugnum => $bugnum, }; process_template($template, 'newadvisory', $vars, $bugnum, 'adv'); return $file; } sub listadv { my ($advdb, @filter) = @_; my @advlist = keys %{$advdb->{advisories}}; foreach my $f (@filter) { my $l = $advdb->{by_type}{$f} || $advdb->{by_cve}{$f} || $advdb->{by_rel}{$f} || $advdb->{by_media}{$f} || $advdb->{by_src}{$f} || []; my %z; @z{@$l} = (1) x @$l; @advlist = grep { $z{$_} } @advlist; } print map { "$_ . $advdb->{advisories}{$_}{subject}\n" } adv_sort($advdb, @advlist); } sub showadv { my ($advdb, $adv) = @_; if (!$advdb->{advisories}{$adv}) { print STDERR "Cannot find advisory $adv\n"; return undef; } my $template = Template->new( ENCODING => 'utf8', INCLUDE_PATH => $config->{tmpl_dir}, ); my $vars = { config => $config, advisory => $adv, advdb => $advdb, basename => \%basename, tools => \%tools, }; my $advtxt; process_template($template, 'advisory', $vars, \$advtxt, 'txt'); print $advtxt; } 1;