#!/usr/bin/ruby
require 'date'
require 'net/http'
require 'optparse'
require 'thread'
require 'uri'
def get_dates(base, archs_per_distro, optional=true)
r = {}
begin
r['base'] = get_timestamp(base)
rescue Net::OpenTimeout, Timeout::Error, ArgumentError, NoMethodError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::ECONNRESET, IOError, OpenSSL::SSL::SSLError => e
end
archs_per_distro.each{|d, archs|
r[d] = {}
archs.each{|a|
begin
r[d][a] = get_date(base, d, a)
rescue Net::OpenTimeout, Timeout::Error, ArgumentError, NoMethodError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::ECONNRESET, IOError, OpenSSL::SSL::SSLError => e
if !optional then
STDERR.puts "Failed to fetch #{version_url(base, d, a)}"
raise
end
end
}
}
r
end
def get_mirrors
# TODO Get it from the DB
mirrors = []
url = nil
tier1 = false
fetch_url("http://mirrors.mageia.org/").each_line{|l|
if l =~ /rsync.mageia.org/ then
tier1 = true
next
end
if l=~ /<\/tr>/ && !url.nil? then
if tier1 then
mirrors.prepend url
tier1 = false
else
mirrors.append url
end
url = nil
next
end
next unless l =~ /https?:.*>http/
# No need to check twice mirrors available in http + https
if !url.nil? && url =~ /https:/ && l =~ /https:\/\//
# Skip http:// if https:// already seen for current mirror
# If the are in the other order http one will just be replaced
next
end
url = l.sub(/<a href="(http[^"]*)".*\n/, '\1')
url += "/" unless url =~ /\/$/
}
mirrors
end
def fetch_url(url, redirect_limit = 3)
return if redirect_limit < 0
if url =~ /^\// then
open(url){|f|
return f.read
}
else
uri = URI.parse(url)
http = Net::HTTP.new(uri.host, uri.port)
http.open_timeout = 9
http.read_timeout = 9
if uri.scheme == 'https' then
http.use_ssl = true
end
# Ruby 1.8.7 doesn't set a default User-Agent which causes at
# least one mirror to return 403
response = http.get(uri.path, {'User-Agent' => 'check_mirrors'})
case response
when Net::HTTPSuccess then
return response.body
when Net::HTTPRedirection then
location = response['location']
# Make location absolute if it was not
if location =~ /:\/\// then
fetch_url(location, redirect_limit - 1)
else
uri.path = location
fetch_url(uri.to_s, redirect_limit - 1)
end
end
end
end
def timestamp_url(url)
"#{url}mageia_timestamp"
end
def get_timestamp(url)
ti = fetch_url(timestamp_url(url)).to_i
if ti == 0 then
return nil
end
return DateTime.strptime(ti.to_s, '%s')
end
def parse_version(version)
date = version.sub(/.* (........ ..:..)$/, '\1').rstrip
DateTime.strptime(date, '%Y%m%d %H:%M')
end
def version_url(url, distrib, arch)
"#{url}distrib/#{distrib}/#{arch}/VERSION"
end
def get_date(url, distrib, arch)
return parse_version(fetch_url(version_url(url, distrib, arch)))
end
def format_age(ref_time, time)
return " <td class='broken'>X</td>" unless ref_time and time
diff = ref_time - time
cls = 'broken'
if diff == 0 then
cls = 'ok'
elsif diff < 0.5 then
cls = 'almost'
elsif diff < 2 then
cls = 'bad'
end
if cls == 'ok' then
return " <td class='#{cls}'> </td>"
else
return " <td class='#{cls}'>#{time.strftime("%F %R")}</td>"
end
end
def print_output(archs_per_distro, mirrors, ref_times, times)
puts "<html><head><title>Mageia Mirror Status #{Time.now.strftime("%F")}</title>
<link rel=\"icon\" type=\"image/png\" href=\"//www.mageia.org/g/favicon.png\">
<style>
td.broken {background-color:#FF0033;}
td.bad {background-color:#FF9933;}
td.almost {background-color:#CCFF66;}
td.ok {background-color:#00FF66;}
td {text-align:center;}
td.name {text-align:left;}
td.sep {width:12px;}
table.legend td {padding:4px;}
th {background-color:#EEEEEE;}
</style>
</head>
<body>"
puts "Last checked on #{Time.now.strftime("%F %R %Z")}<br/>"
puts "<table class='legend'><tr><td class='ok'>Up to date</td><td class='almost'>Less than 12h old</td><td class='bad'>Less than 2 days old</td><td class='broken'>Old or broken</td></tr></table>"
puts "<table><thead>"
puts "<tr><td/>"
puts "<td/><th>Base directory</th>"
archs_per_distro.each{|d, archs|
nb_arches = archs.size
puts " <td/><th colspan='#{nb_arches}'>#{d}</th>"
}
puts "</tr>"
puts "<tr><td/><td/><td/>"
archs_per_distro.each{|d, archs|
puts " <td class='sep' />"
archs.each{|a|
puts " <th>#{a}</th>"
}
}
puts "</tr></thead>"
puts "<tbody>"
puts "<tr><td class='name'>Reference</td>"
puts " <td class='sep' />"
puts " <td>#{!ref_times['base'].nil? ? ref_times['base'].strftime("%F %R") : "?"}</td>"
archs_per_distro.each{|d, archs|
puts " <td class='sep' />"
archs.each{|a|
puts " <td>#{ref_times[d][a].strftime("%F %R")}</td>"
}
}
puts "</tr>"
mirrors.each{|u|
puts "<tr><td class='name'><a href='#{u}'>#{u}</a></td>"
puts " <td class='sep' />"
puts format_age(ref_times['base'], times[u]['base'])
archs_per_distro.each{|d, archs|
puts " <td class='sep' />"
archs.each{|a|
puts format_age(ref_times[d][a], times[u][d][a])
}
}
puts "</tr>"
}
puts "</tbody></table>"
puts "</body></html>"
end
# Defaults
ref = 'http://repository.mageia.org/'
archs_per_distro = {
'9' => ['i586', 'x86_64', 'armv7hl', 'aarch64'],
'8' => ['i586', 'x86_64', 'armv7hl', 'aarch64'],
'cauldron' => ['i586', 'x86_64', 'armv7hl', 'aarch64']
}
parallel = 8
OptionParser.new {|opts|
opts.banner = "Usage: #{$0} [options]"
opts.on("--repository URL",
"Reference repository. Default: #{ref}") {
|url| ref = url
}
opts.on("--parallel n", Integer,
"Max number of parallel connections. Default: #{parallel}") {
|n| $parallel = n
}
opts.on("--output file",
"Write output into given file. Default to STDOUT") {
|f| $stdout.reopen(f, "w")
}
}.parse!
# Get dates from the reference repository, and fail if some requested distros
# or archs are missing
ref_times = get_dates(ref, archs_per_distro, false)
# Get the list of mirror URLs to check
mirrors = get_mirrors
workqueue = Queue.new
times = {}
# Create all the thread and have them loop on the work queue
threads = (1..parallel).map{|n|
Thread.new {
loop do
u = workqueue.pop
break if u == :exit
times[u] = get_dates(u, archs_per_distro)
end
}
}
# Push all mirrors into the queue
mirrors.each{|u|
workqueue << u
}
# Get all the threads to exit after all the work is done
parallel.times{|i|
workqueue << :exit
}
# Wait for the threads to exit
threads.each{|t|
t.join
}
# Generate output
print_output(archs_per_distro, mirrors, ref_times, times)