mga-mirrors: add check_mirrors_status script and cron

It was manually installed on alamut, as /usr/local/bin/check_mirrors, and had a custom crontab entry for user nobody. This also renames the script as check_mirrors_status, to avoid confusion with /usr/bin/check_mirror from the mga-mirrors package. From unknown author (I bet pterjan).
author: Olivier Blin <dev@blino.org> 2017-04-08 17:43:26 +0200
committer: Olivier Blin <dev@blino.org> 2017-04-08 17:43:44 +0200
commit: 1b7ff1fc5804532ccfe815a27c71feca68b7bf6b (patch)
tree: 07836f602c8c4a82cea67791ff82dbaea0212386 /modules
parent: c4ba370241185f29683f11cfa3bfb688b98869ef (diff)
download: puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.tar
puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.tar.gz
puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.tar.bz2
puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.tar.xz
puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.zip
3 files changed, 229 insertions, 0 deletions
diff --git a/modules/mga-mirrors/manifests/init.pp b/modules/mga-mirrors/manifests/init.pp
index 78439c35..536b4662 100644
--- a/modules/mga-mirrors/manifests/init.pp
+++ b/modules/mga-mirrors/manifests/init.pp
@@ -45,4 +45,9 @@ class mga-mirrors {
         ensure => directory,
 	owner  => 'nobody',
     }
+
+    file { '/usr/local/bin/check_mirrors_status':
+        mode   => '0755',
+        source => 'puppet:///modules/mga-mirrors/check_mirrors_status',
+    }
 }
diff --git a/modules/mga-mirrors/templates/check_mirrors_status b/modules/mga-mirrors/templates/check_mirrors_status
new file mode 100755
index 00000000..06a17c2a
--- /dev/null
+++ b/modules/mga-mirrors/templates/check_mirrors_status
@@ -0,0 +1,223 @@
+#!/usr/bin/ruby
+
+require 'date'
+require 'net/http'
+require 'optparse'
+require 'thread'
+require 'uri'
+
+def get_dates(base, distros, archs, optional=true)
+  r = {}
+  distros.each{|d|
+    r[d] = {}
+    archs.each{|a|
+      next if d == '5' and a == 'armv5tl'
+      begin
+        r[d][a] = get_date(base, d, a)
+      rescue Timeout::Error, ArgumentError, NoMethodError, Errno::EHOSTUNREACH => e
+        if !optional then
+          raise
+        end
+      end
+    }
+  }
+  r
+end
+
+def get_mirrors
+  # TODO Get it from the DB
+  mirrors = []
+  fetch_url("http://mirrors.mageia.org/").each_line{|l|
+    next unless l =~ /http:.*>http/ 
+    mirrors << l.sub(/<a href="(http[^"]*)".*\n/, '\1')
+  }
+  mirrors
+end
+
+def fetch_url(url, redirect_limit = 3)
+  return if redirect_limit < 0
+  if url =~ /^\// then
+    open(url){|f|
+      return f.read
+    }
+  else
+    uri = URI.parse(url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.open_timeout = 2
+    http.read_timeout = 2
+    # Ruby 1.8.7 doesn't set a default User-Agent which causes at
+    # least one mirror to return 403
+    response = http.get(uri.path, {'User-Agent' => 'check_mirrors'})
+    case response
+    when Net::HTTPSuccess then
+      return response.body
+    when Net::HTTPRedirection then
+      location = response['location']
+      # Make location absolute if it was not
+      if location =~ /:\/\// then
+        fetch_url(location, redirect_limit - 1)
+      else
+        uri.path = location
+        fetch_url(uri.to_s, redirect_limit - 1)
+      end
+    end
+  end
+end
+
+def parse_version(version)
+  date = version.sub(/.* (........ ..:..)$/, '\1').rstrip
+  DateTime.strptime(date, '%Y%m%d %H:%M')
+end
+
+def get_date(url, distrib, arch)
+  return parse_version(fetch_url("#{url}/distrib/#{distrib}/#{arch}/VERSION"))
+end
+
+def print_output(distros, archs, mirrors, ref_times, times)
+  puts "<html>
+<style>
+td.broken {background-color:#FF0033;}
+td.bad {background-color:#FF9933;}
+td.almost {background-color:#CCFF66;}
+td.ok {background-color:#00FF66;}
+
+td {text-align:center;}
+td.name {text-align:left;}
+
+td.sep {width:12px;}
+
+th {background-color:#EEEEEE;}
+</style>
+<body>"
+  puts "Last checked on #{Time.now.strftime("%F %R")}<br/>"
+  puts "<table cellpadding='4px'><tr><td class='ok'>Up to date</td><td class='almost'>Less than 12h old</td><td class='bad'>Less than 2 days old</td><td class='broken'>Old or broken</td></tr></table>"
+  puts "<table><thead>"
+  puts "<tr><td/>"
+  distros.each{|d|
+	 nb_arches = archs.size
+	 nb_arches -= 1 if d == '5' # 'armv5tl'
+	 puts "  <td/><th colspan='#{nb_arches}'>#{d}</th>"}
+  puts "</tr>"
+  puts "<tr><td/>"
+  distros.each{|d|
+	  puts "  <td class='sep' />"
+	  archs.each{|a| 
+		next if d == '5' and a == 'armv5tl'
+		puts "  <th>#{a}</th>"
+	  }
+  }
+  puts "</tr></thead>"
+  puts "<tbody>"
+  puts "<tr><td class='name'>Reference</td>"
+  distros.each{|d|
+    puts "  <td class='sep' />"
+    archs.each{|a|
+      next if d == '5' and a == 'armv5tl'
+      puts "  <td>#{ref_times[d][a].strftime("%F %R")}</td>"
+    }
+  }
+  puts "</tr>"
+
+  mirrors.each{|u|
+    puts "<tr><td class='name'><a href='#{u}'>#{u}</a></td>"
+    distros.each{|d|
+      puts "  <td class='sep' />"
+      archs.each{|a|
+        next if d == '5' and a == 'armv5tl'
+        if times[u][d][a] != nil then
+          cls = 'broken'
+          diff = ref_times[d][a] - times[u][d][a]
+          if diff == 0 then
+            cls = 'ok'
+          elsif diff < 0.5 then
+            cls = 'almost'
+          elsif diff < 2 then
+            cls = 'bad'
+          end
+          if cls == 'ok' then
+            puts "  <td class='#{cls}'>&nbsp;</td>"
+          else
+            puts "  <td class='#{cls}'>#{times[u][d][a].strftime("%F %R")}</td>"
+          end
+        else
+          puts "  <td class='broken'>X</td>"
+        end
+      }
+    }
+    puts "</tr>"
+  }
+  puts "</tbody></table>"
+  puts "</body></html>"
+end
+
+
+
+# Defaults
+ref = 'http://repository.mageia.org/'
+distros = ['5', 'cauldron']
+archs = ['i586', 'x86_64', 'armv5tl']
+parallel = 8
+
+OptionParser.new {|opts|
+  opts.banner = "Usage: #{$0} [options]"
+  opts.on("--repository URL",
+          "Reference repository. Default: #{ref}") {
+    |url| ref = url
+  }
+  opts.on("--distros x,y,z", Array,
+          "List of distributions to check. Default: #{distros}") {
+    |list| distros = list
+  }
+  opts.on("--archs x,y,z", Array,
+          "List of architectures to check. Default: #{archs}") {
+    |list| archs = list
+  }
+  opts.on("--parallel n", Integer,
+	  "Max number of parallel connections. Default: #{parallel}") {
+    |n| $parallel = n
+  }
+  opts.on("--output file",
+	  "Write output into given file. Default to STDOUT") {
+    |f| $stdout.reopen(f, "w")
+  }
+}.parse!
+
+# Get dates from the reference repository, and fail if some requested distros 
+# or archs are missing
+ref_times = get_dates(ref, distros, archs, false)
+
+# Get the list of mirror URLs to check
+mirrors = get_mirrors
+
+workqueue = Queue.new
+times = {}
+
+# Create all the thread and have them loop on the work queue
+threads = (1..parallel).map{|n|
+  Thread.new {
+    loop do
+      u = workqueue.pop
+      break if u == :exit
+      times[u] = get_dates(u, distros, archs)
+    end
+  }
+}
+
+# Push all mirrors into the queue
+mirrors.each{|u|
+  workqueue << u
+}
+
+# Get all the threads to exit after all the work is done
+parallel.times{|i|
+  workqueue << :exit
+}
+
+# Wait for the threads to exit
+threads.each{|t|
+  t.join
+}
+
+# Generate output
+print_output(distros, archs, mirrors, ref_times, times)
+
diff --git a/modules/mga-mirrors/templates/cron-mga_mirrors b/modules/mga-mirrors/templates/cron-mga_mirrors
index 0f9410a8..96b08bd8 100644
--- a/modules/mga-mirrors/templates/cron-mga_mirrors
+++ b/modules/mga-mirrors/templates/cron-mga_mirrors
@@ -1,2 +1,3 @@
 MAILTO=root
 0 * * * *     apache    /usr/bin/check_mirror > /dev/null 2>&1
+*/20 * * * *     nobody     /usr/local/bin/check_mirrors --output /var/www/mirrors/status.html.tmp && mv -f /var/www/mirrors/status.html.tmp /var/www/mirrors/status.html
author	Olivier Blin <dev@blino.org>	2017-04-08 17:43:26 +0200
committer	Olivier Blin <dev@blino.org>	2017-04-08 17:43:44 +0200
commit	1b7ff1fc5804532ccfe815a27c71feca68b7bf6b (patch)
tree	07836f602c8c4a82cea67791ff82dbaea0212386 /modules
parent	c4ba370241185f29683f11cfa3bfb688b98869ef (diff)
download	puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.tar puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.tar.gz puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.tar.bz2 puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.tar.xz puppet-1b7ff1fc5804532ccfe815a27c71feca68b7bf6b.zip