diff options
| -rw-r--r-- | Makefile | 6 | ||||
| -rw-r--r-- | NEWS | 17 | ||||
| -rw-r--r-- | lib/MGA/Advisories.pm | 54 | ||||
| -rw-r--r-- | tmpl/advisory.html | 1 | ||||
| -rw-r--r-- | tmpl/advisory.json | 1 | ||||
| -rw-r--r-- | tmpl/bugs.json | 3 | ||||
| -rw-r--r-- | tmpl/infos.html | 12 | ||||
| -rw-r--r-- | tmpl/vulns.json | 3 |
8 files changed, 87 insertions, 10 deletions
@@ -1,10 +1,10 @@ -VERSION=0.27 +VERSION=0.30 PROJECTNAME=mga-advisories TARNAME=mgaadvisories BINFILES=mgaadv CFGFILES=mga-advisories.conf -TMPLFILES=tmpl/*.html tmpl/*.txt tmpl/*.adv tmpl/*.rss +TMPLFILES=tmpl/*.html tmpl/*.txt tmpl/*.adv tmpl/*.rss tmpl/*.json STATICFILES=static/* sysconfdir=/etc @@ -30,4 +30,4 @@ install: install -m 644 lib/MGA/Advisories.pm $(DESTDIR)$(perldir)/MGA tar: - git archive HEAD --prefix=$(TARNAME)-$(VERSION)/ -o $(TARNAME)-$(VERSION).tar.xz + git archive HEAD --prefix=$(TARNAME)-$(VERSION)/ | xz > $(TARNAME)-$(VERSION).tar.xz @@ -1,4 +1,18 @@ -Version X +Version 0.X + +- ensure .adv file ends with newline when publishing +- show how to get the OSV format advisories on the infos page + +Version 0.30 + +- try to fix publish-all +- include a last modified date in advisories + +Version 0.29 + +- install the new JSON templates + +Version 0.28 - template: change default CVE year to 2024 - use https: links where possible @@ -6,6 +20,7 @@ Version X - have 'mksite' write JSON output files as well - 'mksite' now runs twice as fast - cross-check that at least one SRPM is listed in the advisory on publish +- make tar now compresses the file Version 0.27 diff --git a/lib/MGA/Advisories.pm b/lib/MGA/Advisories.pm index dbb7408..3752250 100644 --- a/lib/MGA/Advisories.pm +++ b/lib/MGA/Advisories.pm @@ -5,14 +5,18 @@ use strict; use YAML qw(LoadFile DumpFile Load); use Template; use DateTime; +use DateTime::Format::ISO8601; use Email::Sender::Simple qw(try_to_sendmail); use Email::Simple; use Email::Simple::Creator; +use Fcntl qw(SEEK_END); use HTTP::Request; use LWP::UserAgent; use Parallel::ForkManager; use File::Basename; use XMLRPC::Lite; +use XML::XPath; +use XML::XPath::XMLParser; use Term::ReadKey; #use Data::Dump qw(dd); @@ -169,6 +173,9 @@ sub login_bz { sub get_advisories_from_dir { + # Retrieve last modified dates from SVN + my $modified = get_modified(); + my %advisories; foreach my $advfile (glob "$config->{advisories_dir}/*.adv") { my $adv; @@ -180,9 +187,9 @@ sub get_advisories_from_dir { print $@; next; } + $adv->{ref} = basename($advfile, ".adv"); if (!$adv->{ID}) { next unless $config->{mode} eq 'qa'; - $adv->{ref} = basename($advfile, ".adv"); $adv->{ID} = next_id('TODO', keys %advisories); $adv->{no_save_status} = 1; } @@ -192,6 +199,11 @@ sub get_advisories_from_dir { $advisories{$adv->{ID}} = $adv; my $statusfile = status_file($adv->{ID}); $adv->{status} = -f $statusfile ? LoadFile($statusfile) : {}; + my $fn = $adv->{ref} = basename($advfile); + if (exists $modified->{$fn}) { + # Pull the modified date into the advisory + $adv->{status}{modified} = $modified->{$fn}; + } } return \%advisories; } @@ -221,6 +233,16 @@ sub assign_id { return; } + # Appending the ID later assumes that the file ends in a newline + open(my $fha, "<", $advfile); + seek($fha, -1, SEEK_END); + my $c = <$fha>; + close($fha); + if ($c ne "\n") { + print STDERR "$advname missing newline at end of file\n"; + return; + } + my $adv = LoadFile($advfile); if ($adv->{ID}) { print STDERR "$advname already has an ID assigned: $adv->{ID}\n"; @@ -446,7 +468,7 @@ sub assign_ids { my %advdb; $advdb{advisories} = get_advisories_from_dir(); sort_advisories(\%advdb); - output_pages(); + output_pages(\%advdb); # We will have exited by now in the event of e.g. a Yaml or processing error @@ -619,8 +641,33 @@ sub process_template { } } +# Get the last modified date for each advisory file from SVN +sub get_modified { + my $xml = `svn status -v --xml`; + my $xp = XML::XPath->new(xml => $xml); + my $nodeset = $xp->find('/status/target/entry'); + my %modified; + foreach my $node ($nodeset->get_nodelist) { + my $path = $node->findvalue('@path')->value(); + my $datez = $node->findvalue('wc-status/commit/date')->value(); + if ($path and $datez) { + my $timestamp = DateTime::Format::ISO8601->parse_datetime($datez); + $modified{$path} = $timestamp->epoch; + } + } + return \%modified; +} + # Max 10 processes for processing templates my $pm = Parallel::ForkManager->new(10); +$pm->run_on_finish(\¶llel_finish); +my $parallelerror = 0; + +# Store error flag from forked process +sub parallel_finish { + my ($pid, $exit_code, $ident) = @_; + $parallelerror |= $exit_code; +} # Run process_template in its own process. The process creation overhead is # high, so this only makes sense for templates that interate over all or most @@ -633,9 +680,10 @@ sub parallel_process_template { } } -# Wait for all processes to finish +# Wait for all processes to finish & die if any returned an error sub parallel_complete { $pm->wait_all_children; + die "Error writing output" if $parallelerror; } sub output_pages { diff --git a/tmpl/advisory.html b/tmpl/advisory.html index a5cbb52..dda79a4 100644 --- a/tmpl/advisory.html +++ b/tmpl/advisory.html @@ -15,6 +15,7 @@ <h2>[% adv.subject %]</h2> Publication date: [% date.format(adv.status.published, format => '%d %b %Y', gmt => 1) %]<br /> + Modification date: [% date.format(adv.status.modified, format => '%d %b %Y', gmt => 1) %]<br /> Type: [% adv.type %]<br /> Affected Mageia releases : [% SET sep = '' %] diff --git a/tmpl/advisory.json b/tmpl/advisory.json index 475012f..ab46f2b 100644 --- a/tmpl/advisory.json +++ b/tmpl/advisory.json @@ -11,6 +11,7 @@ print JSON::encode_json($stash->get($stash->get('var'))); "schema_version": "1.6.2", "id": [% jsonvar('advisory') %], "published": "[% date.format(adv.status.published, format => '%Y-%m-%dT%H:%M:%SZ', gmt => 1) %]", + "modified": "[% date.format(adv.status.modified, format => '%Y-%m-%dT%H:%M:%SZ', gmt => 1) %]", "summary": [% jsonvar('adv.subject') %], "details": [% jsonvar('adv.description') %], [% IF adv.CVE && adv.CVE.list.size != 0 -%] diff --git a/tmpl/bugs.json b/tmpl/bugs.json index 375e7f7..53da837 100644 --- a/tmpl/bugs.json +++ b/tmpl/bugs.json @@ -1,9 +1,10 @@ +[%- USE date -%] [ [%- FOR adv IN advdb.sorted -%] [% USE advid = String(basename.ID(adv)) -%] [% IF advid.search('^MGAA-') -%] [%- "," IF gotone %] -{"id": "[% basename.ID(adv) %]"} +{"id": "[% basename.ID(adv) %]","modified": "[% date.format(advdb.advisories.$adv.status.modified, format => '%Y-%m-%dT%H:%M:%SZ', gmt => 1) %]"} [%- SET gotone = 1 %] [%- END %] [%- END %] diff --git a/tmpl/infos.html b/tmpl/infos.html index 84c41b3..d64ee02 100644 --- a/tmpl/infos.html +++ b/tmpl/infos.html @@ -21,7 +21,7 @@ <div class="section"> <h2>CVE list</h2> - A list of fixed CVE <a href="CVE.html">is available</a>. + A list of fixed CVEs <a href="CVE.html">is available</a>. </div> <div class="section"> @@ -41,6 +41,16 @@ </div> <div class="section"> + <h2>OSV Format</h2> + Advisories are also available in <a href="https://ossf.github.io/osv-schema/">OSV format</a>. + Each advisory is available from <em>https://advisories.mageia.org/<MGASA-ID>.json</em> + where <em><MGASA-ID></em> is the advisory ID. + An <a href="https://advisories.mageia.org/vulns.json">index to all security advisories</a> + is also available (in JSON format) as is an + <a href="https://advisories.mageia.org/bugs.json">index to bug advisories</a>. + </div> + + <div class="section"> <h2>Source code</h2> Source code for the tool used to generate this website is <a href="https://gitweb.mageia.org/software/infrastructure/mgaadvisories/">available on git</a>. diff --git a/tmpl/vulns.json b/tmpl/vulns.json index aadcd05..790f88a 100644 --- a/tmpl/vulns.json +++ b/tmpl/vulns.json @@ -1,9 +1,10 @@ +[%- USE date -%] [ [%- FOR adv IN advdb.sorted -%] [% USE advid = String(basename.ID(adv)) -%] [% IF advid.search('^MGASA-') -%] [%- "," IF gotone %] -{"id": "[% basename.ID(adv) %]"} +{"id": "[% basename.ID(adv) %]","modified": "[% date.format(advdb.advisories.$adv.status.modified, format => '%Y-%m-%dT%H:%M:%SZ', gmt => 1) %]"} [%- SET gotone = 1 %] [%- END %] [%- END %] |