diff options
Diffstat (limited to 'modules/mga-mirrors')
| -rwxr-xr-x | modules/mga-mirrors/files/check_mirrors_status | 271 | ||||
| -rw-r--r-- | modules/mga-mirrors/manifests/init.pp | 61 | ||||
| -rw-r--r-- | modules/mga-mirrors/templates/cron-mga_mirrors | 2 | ||||
| -rw-r--r-- | modules/mga-mirrors/templates/mga-mirrors.ini | 4 |
4 files changed, 321 insertions, 17 deletions
diff --git a/modules/mga-mirrors/files/check_mirrors_status b/modules/mga-mirrors/files/check_mirrors_status new file mode 100755 index 00000000..9c00ac8d --- /dev/null +++ b/modules/mga-mirrors/files/check_mirrors_status @@ -0,0 +1,271 @@ +#!/usr/bin/ruby + +require 'date' +require 'net/http' +require 'optparse' +require 'thread' +require 'uri' + +def get_dates(base, archs_per_distro, optional=true) + r = {} + begin + r['base'] = get_timestamp(base) + rescue Net::OpenTimeout, Timeout::Error, ArgumentError, NoMethodError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::ECONNRESET, IOError, OpenSSL::SSL::SSLError => e + end + + archs_per_distro.each{|d, archs| + r[d] = {} + archs.each{|a| + begin + r[d][a] = get_date(base, d, a) + rescue Net::OpenTimeout, Timeout::Error, ArgumentError, NoMethodError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::ECONNRESET, IOError, OpenSSL::SSL::SSLError => e + if !optional then + STDERR.puts "Failed to fetch #{version_url(base, d, a)}" + raise + end + end + } + } + r +end + +def get_mirrors + # TODO Get it from the DB + mirrors = [] + url = nil + tier1 = false + fetch_url("https://mirrors.mageia.org/").each_line{|l| + if l =~ /rsync.mageia.org/ then + tier1 = true + next + end + if l=~ /<\/tr>/ && !url.nil? then + if tier1 then + mirrors.prepend url + tier1 = false + else + mirrors.append url + end + url = nil + next + end + next unless l =~ /https?:.*>http/ + # No need to check twice mirrors available in http + https + if !url.nil? && url =~ /https:/ && l =~ /https:\/\// + # Skip http:// if https:// already seen for current mirror + # If the are in the other order http one will just be replaced + next + end + url = l.sub(/<a href="(http[^"]*)".*\n/, '\1') + url += "/" unless url =~ /\/$/ + } + mirrors +end + +def fetch_url(url, redirect_limit = 3) + return if redirect_limit < 0 + if url =~ /^\// then + open(url){|f| + return f.read + } + else + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.open_timeout = 30 + http.read_timeout = 30 + if uri.scheme == 'https' then + http.use_ssl = true + end + # Ruby 1.8.7 doesn't set a default User-Agent which causes at + # least one mirror to return 403 + response = http.get(uri.path, {'User-Agent' => 'check_mirrors'}) + case response + when Net::HTTPSuccess then + return response.body + when Net::HTTPRedirection then + location = response['location'] + # Make location absolute if it was not + if location =~ /:\/\// then + fetch_url(location, redirect_limit - 1) + else + uri.path = location + fetch_url(uri.to_s, redirect_limit - 1) + end + end + end +end + +def timestamp_url(url) + "#{url}mageia_timestamp" +end + +def get_timestamp(url) + ti = fetch_url(timestamp_url(url)).to_i + if ti == 0 then + return nil + end + return DateTime.strptime(ti.to_s, '%s') +end + +def parse_version(version) + date = version.sub(/.* (........ ..:..)$/, '\1').rstrip + DateTime.strptime(date, '%Y%m%d %H:%M') +end + +def version_url(url, distrib, arch) + "#{url}distrib/#{distrib}/#{arch}/VERSION" +end + +def get_date(url, distrib, arch) + return parse_version(fetch_url(version_url(url, distrib, arch))) +end + +def format_age(ref_time, time) + return " <td class='broken'>X</td>" unless ref_time and time + + diff = ref_time - time + cls = 'broken' + if diff == 0 then + cls = 'ok' + elsif diff < 0.5 then + cls = 'almost' + elsif diff < 2 then + cls = 'bad' + end + if cls == 'ok' then + return " <td class='#{cls}'> </td>" + else + return " <td class='#{cls}'>#{time.strftime("%F %R")}</td>" + end +end + +def print_output(archs_per_distro, mirrors, ref_times, times) + puts "<html><head><title>Mageia Mirror Status #{Time.now.utc.strftime("%F")}</title> +<link rel=\"icon\" type=\"image/png\" href=\"//www.mageia.org/g/favicon.png\"> +<style> +td.broken {background-color:#FF0033;} +td.bad {background-color:#FF9933;} +td.almost {background-color:#CCFF66;} +td.ok {background-color:#00FF66;} + +td {text-align:center;} +td.name {text-align:left;} + +td.sep {width:12px;} +table.legend td {padding:4px;} + +th {background-color:#EEEEEE;} +</style> +</head> +<body>" + puts "Last checked on #{Time.now.utc.strftime("%F %R %Z")}<br/>" + puts "<table class='legend'><tr><td class='ok'>Up to date</td><td class='almost'>Less than 12h old</td><td class='bad'>Less than 2 days old</td><td class='broken'>Old or broken</td></tr></table>" + puts "<table><thead>" + puts "<tr><td/>" + puts "<td/><th>Base directory</th>" + archs_per_distro.each{|d, archs| + nb_arches = archs.size + puts " <td/><th colspan='#{nb_arches}'>#{d}</th>" + } + puts "</tr>" + puts "<tr><td/><td/><td/>" + archs_per_distro.each{|d, archs| + puts " <td class='sep' />" + archs.each{|a| + puts " <th>#{a}</th>" + } + } + puts "</tr></thead>" + puts "<tbody>" + puts "<tr><td class='name'>Reference</td>" + puts " <td class='sep' />" + puts " <td>#{!ref_times['base'].nil? ? ref_times['base'].strftime("%F %R") : "?"}</td>" + archs_per_distro.each{|d, archs| + puts " <td class='sep' />" + archs.each{|a| + puts " <td>#{ref_times[d][a].strftime("%F %R")}</td>" + } + } + puts "</tr>" + + mirrors.each{|u| + puts "<tr><td class='name'><a href='#{u}'>#{u}</a></td>" + puts " <td class='sep' />" + puts format_age(ref_times['base'], times[u]['base']) + archs_per_distro.each{|d, archs| + puts " <td class='sep' />" + archs.each{|a| + puts format_age(ref_times[d][a], times[u][d][a]) + } + } + puts "</tr>" + } + puts "</tbody></table>" + puts "</body></html>" +end + + + +# Defaults +ref = 'http://repository.mageia.org/' +archs_per_distro = { + 'cauldron' => ['i686', 'x86_64', 'armv7hl', 'aarch64'], + '9' => ['i586', 'x86_64', 'armv7hl', 'aarch64'] +} +parallel = 8 + +OptionParser.new {|opts| + opts.banner = "Usage: #{$0} [options]" + opts.on("--repository URL", + "Reference repository. Default: #{ref}") { + |url| ref = url + } + opts.on("--parallel n", Integer, + "Max number of parallel connections. Default: #{parallel}") { + |n| $parallel = n + } + opts.on("--output file", + "Write output into given file. Default to STDOUT") { + |f| $stdout.reopen(f, "w") + } +}.parse! + +# Get dates from the reference repository, and fail if some requested distros +# or archs are missing +ref_times = get_dates(ref, archs_per_distro, false) + +# Get the list of mirror URLs to check +mirrors = get_mirrors + +workqueue = Queue.new +times = {} + +# Create all the thread and have them loop on the work queue +threads = (1..parallel).map{|n| + Thread.new { + loop do + u = workqueue.pop + break if u == :exit + times[u] = get_dates(u, archs_per_distro) + end + } +} + +# Push all mirrors into the queue +mirrors.each{|u| + workqueue << u +} + +# Get all the threads to exit after all the work is done +parallel.times{|i| + workqueue << :exit +} + +# Wait for the threads to exit +threads.each{|t| + t.join +} + +# Generate output +print_output(archs_per_distro, mirrors, ref_times, times) + diff --git a/modules/mga-mirrors/manifests/init.pp b/modules/mga-mirrors/manifests/init.pp index f602a47e..4b8b5552 100644 --- a/modules/mga-mirrors/manifests/init.pp +++ b/modules/mga-mirrors/manifests/init.pp @@ -1,23 +1,54 @@ class mga-mirrors { - - $vhost = "mirrors.$domain" - package { 'mga-mirrors': - ensure => installed + $vhost = "mirrors.${::domain}" + $mirrors_dir = '/var/www/mirrors' + + package { 'mga-mirrors': } + + apache::vhost::catalyst_app { $vhost: + script => '/usr/bin/mga_mirrors_fastcgi.pl', + require => Package['mga-mirrors'], + aliases => { + '/status' => '/var/www/mirrors/status.html', + } + } + + apache::vhost::catalyst_app { "ssl_${vhost}": + script => '/usr/bin/mga_mirrors_fastcgi.pl', + require => Package['mga-mirrors'], + vhost => $vhost, + use_ssl => true, + aliases => { + '/status' => '/var/www/mirrors/status.html', + }, + } + + $pgsql_password = extlookup('mga_mirror_pgsql','x') + + postgresql::remote_db_and_user { 'mirrors': + password => $pgsql_password, + description => 'Mirrors database', + } + + file { '/etc/mga-mirrors.ini': + group => 'apache', + mode => '0640', + content => template('mga-mirrors/mga-mirrors.ini'), + require => Package['mga-mirrors'] + } + + file { '/etc/cron.d/check_mga_mirrors': + content => template('mga-mirrors/cron-mga_mirrors'), + require => Package['mga-mirrors'] } - apache::vhost_catalyst_app { $vhost: - script => "/usr/bin/mga_mirrors_fastcgi.pl" + file { $mirrors_dir: + ensure => directory, + owner => 'nobody', } - $password = extlookup("mga_mirror_password") - - file { "mga-mirrors.ini": - path => "/etc/mga-mirrors.ini", - ensure => "present", - owner => root, - group => apache, - mode => 640, - content => template("mga-mirrors/mga-mirrors.ini") + file { '/usr/local/bin/check_mirrors_status': + mode => '0755', + source => 'puppet:///modules/mga-mirrors/check_mirrors_status', } } diff --git a/modules/mga-mirrors/templates/cron-mga_mirrors b/modules/mga-mirrors/templates/cron-mga_mirrors new file mode 100644 index 00000000..7236be04 --- /dev/null +++ b/modules/mga-mirrors/templates/cron-mga_mirrors @@ -0,0 +1,2 @@ +MAILTO=root +*/20 * * * * nobody /usr/local/bin/check_mirrors_status --output /var/www/mirrors/status.html.tmp && mv -f /var/www/mirrors/status.html.tmp /var/www/mirrors/status.html diff --git a/modules/mga-mirrors/templates/mga-mirrors.ini b/modules/mga-mirrors/templates/mga-mirrors.ini index 973c65fd..b438edd1 100644 --- a/modules/mga-mirrors/templates/mga-mirrors.ini +++ b/modules/mga-mirrors/templates/mga-mirrors.ini @@ -1,4 +1,4 @@ [db] -pgconn=host=pgsql.<%= domain %>;dbname=mirrors +pgconn=host=pg.<%= @domain %>;dbname=mirrors user=mirrors -password=<%= password %> +password=<%= @pgsql_password %> |