Read advisories from disk in parallel

The parsing overhead is now spread over multiple cores when available,
dramatically reducing the time to read them all. mgaadv list is twice as
fast now on one test machine, for example.

2 files changed, 32 insertions, 9 deletions

diff --git a/NEWS b/NEWS
index 93056da..e29cfa4 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+Version 0.X
+
+- loading all advisories is much faster on multicore machines
+
 Version 0.31
 
 - ensure .adv file ends with newline when publishing
diff --git a/lib/MGA/Advisories.pm b/lib/MGA/Advisories.pm
index 3752250..ea32f90 100644
--- a/lib/MGA/Advisories.pm
+++ b/lib/MGA/Advisories.pm
@@ -12,6 +12,7 @@ use Email::Simple::Creator;
 use Fcntl qw(SEEK_END);
 use HTTP::Request;
 use LWP::UserAgent;
+use MCE::Map;
 use Parallel::ForkManager;
 use File::Basename;
 use XMLRPC::Lite;
@@ -171,23 +172,36 @@ sub login_bz {
     return 0;
 }
 
+# Load the advisory and its status and return both the filename and advisory contents
+sub read_adv {
+    my ($advfile) = @_;
+    my $adv;
+    eval {
+        $adv = LoadFile($advfile);
+    };
+    if ($adv) {
+        $adv->{ref} = basename($advfile, ".adv");
+        if ($adv->{ID}) {
+            my $statusfile = status_file($adv->{ID});
+            $adv->{status} = -f $statusfile ? LoadFile($statusfile) : {};
+        }
+    }
+    return [$advfile, $adv];
+}
 
 sub get_advisories_from_dir {
     # Retrieve last modified dates from SVN
     my $modified = get_modified();
 
     my %advisories;
-    foreach my $advfile (glob "$config->{advisories_dir}/*.adv") {
-        my $adv;
-        eval {
-            $adv = LoadFile($advfile);
-        };
-        if ($@) {
+    # Read all advisories in parallel
+    foreach my $advdata (mce_map {read_adv($_)} glob "$config->{advisories_dir}/*.adv") {
+        my ($advfile, $adv) = @$advdata;
+        if (!$adv) {
             print "Failed to load $advfile\n";
             print $@;
             next;
         }
-        $adv->{ref} = basename($advfile, ".adv");
         if (!$adv->{ID}) {
             next unless $config->{mode} eq 'qa';
             $adv->{ID} = next_id('TODO', keys %advisories);
@@ -197,8 +211,13 @@ sub get_advisories_from_dir {
         report_exit("Unknown type $adv->{type}") unless
                 $config->{advisory_types}{$adv->{type}};
         $advisories{$adv->{ID}} = $adv;
-        my $statusfile = status_file($adv->{ID});
-        $adv->{status} = -f $statusfile ? LoadFile($statusfile) : {};
+        if (!$adv->{status}) {
+            # If it isn't already loaded
+            my $statusfile = status_file($adv->{ID});
+            $adv->{status} = -f $statusfile ? LoadFile($statusfile) : {};
+        }
+        # TODO: this changes the ref set previously to include the extension
+        # this time. Is that deliberate?
         my $fn = $adv->{ref} = basename($advfile);
         if (exists $modified->{$fn}) {
             # Pull the modified date into the advisory