From 1b7ff1fc5804532ccfe815a27c71feca68b7bf6b Mon Sep 17 00:00:00 2001 From: Olivier Blin Date: Sat, 8 Apr 2017 17:43:26 +0200 Subject: mga-mirrors: add check_mirrors_status script and cron It was manually installed on alamut, as /usr/local/bin/check_mirrors, and had a custom crontab entry for user nobody. This also renames the script as check_mirrors_status, to avoid confusion with /usr/bin/check_mirror from the mga-mirrors package. From unknown author (I bet pterjan). --- modules/mga-mirrors/manifests/init.pp | 5 + modules/mga-mirrors/templates/check_mirrors_status | 223 +++++++++++++++++++++ modules/mga-mirrors/templates/cron-mga_mirrors | 1 + 3 files changed, 229 insertions(+) create mode 100755 modules/mga-mirrors/templates/check_mirrors_status (limited to 'modules') diff --git a/modules/mga-mirrors/manifests/init.pp b/modules/mga-mirrors/manifests/init.pp index 78439c35..536b4662 100644 --- a/modules/mga-mirrors/manifests/init.pp +++ b/modules/mga-mirrors/manifests/init.pp @@ -45,4 +45,9 @@ class mga-mirrors { ensure => directory, owner => 'nobody', } + + file { '/usr/local/bin/check_mirrors_status': + mode => '0755', + source => 'puppet:///modules/mga-mirrors/check_mirrors_status', + } } diff --git a/modules/mga-mirrors/templates/check_mirrors_status b/modules/mga-mirrors/templates/check_mirrors_status new file mode 100755 index 00000000..06a17c2a --- /dev/null +++ b/modules/mga-mirrors/templates/check_mirrors_status @@ -0,0 +1,223 @@ +#!/usr/bin/ruby + +require 'date' +require 'net/http' +require 'optparse' +require 'thread' +require 'uri' + +def get_dates(base, distros, archs, optional=true) + r = {} + distros.each{|d| + r[d] = {} + archs.each{|a| + next if d == '5' and a == 'armv5tl' + begin + r[d][a] = get_date(base, d, a) + rescue Timeout::Error, ArgumentError, NoMethodError, Errno::EHOSTUNREACH => e + if !optional then + raise + end + end + } + } + r +end + +def get_mirrors + # TODO Get it from the DB + mirrors = [] + fetch_url("http://mirrors.mageia.org/").each_line{|l| + next unless l =~ /http:.*>http/ + mirrors << l.sub(/ 'check_mirrors'}) + case response + when Net::HTTPSuccess then + return response.body + when Net::HTTPRedirection then + location = response['location'] + # Make location absolute if it was not + if location =~ /:\/\// then + fetch_url(location, redirect_limit - 1) + else + uri.path = location + fetch_url(uri.to_s, redirect_limit - 1) + end + end + end +end + +def parse_version(version) + date = version.sub(/.* (........ ..:..)$/, '\1').rstrip + DateTime.strptime(date, '%Y%m%d %H:%M') +end + +def get_date(url, distrib, arch) + return parse_version(fetch_url("#{url}/distrib/#{distrib}/#{arch}/VERSION")) +end + +def print_output(distros, archs, mirrors, ref_times, times) + puts " + +" + puts "Last checked on #{Time.now.strftime("%F %R")}
" + puts "
Up to dateLess than 12h oldLess than 2 days oldOld or broken
" + puts "" + puts ""} + puts "" + puts "" + } + } + puts "" + puts "" + puts "" + distros.each{|d| + puts " " + } + } + puts "" + + mirrors.each{|u| + puts "" + distros.each{|d| + puts " " + else + puts " " + end + else + puts " " + end + } + } + puts "" + } + puts "
" + distros.each{|d| + nb_arches = archs.size + nb_arches -= 1 if d == '5' # 'armv5tl' + puts " #{d}
" + distros.each{|d| + puts " " + archs.each{|a| + next if d == '5' and a == 'armv5tl' + puts " #{a}
Reference" + archs.each{|a| + next if d == '5' and a == 'armv5tl' + puts " #{ref_times[d][a].strftime("%F %R")}
#{u}" + archs.each{|a| + next if d == '5' and a == 'armv5tl' + if times[u][d][a] != nil then + cls = 'broken' + diff = ref_times[d][a] - times[u][d][a] + if diff == 0 then + cls = 'ok' + elsif diff < 0.5 then + cls = 'almost' + elsif diff < 2 then + cls = 'bad' + end + if cls == 'ok' then + puts "  #{times[u][d][a].strftime("%F %R")}X
" + puts "" +end + + + +# Defaults +ref = 'http://repository.mageia.org/' +distros = ['5', 'cauldron'] +archs = ['i586', 'x86_64', 'armv5tl'] +parallel = 8 + +OptionParser.new {|opts| + opts.banner = "Usage: #{$0} [options]" + opts.on("--repository URL", + "Reference repository. Default: #{ref}") { + |url| ref = url + } + opts.on("--distros x,y,z", Array, + "List of distributions to check. Default: #{distros}") { + |list| distros = list + } + opts.on("--archs x,y,z", Array, + "List of architectures to check. Default: #{archs}") { + |list| archs = list + } + opts.on("--parallel n", Integer, + "Max number of parallel connections. Default: #{parallel}") { + |n| $parallel = n + } + opts.on("--output file", + "Write output into given file. Default to STDOUT") { + |f| $stdout.reopen(f, "w") + } +}.parse! + +# Get dates from the reference repository, and fail if some requested distros +# or archs are missing +ref_times = get_dates(ref, distros, archs, false) + +# Get the list of mirror URLs to check +mirrors = get_mirrors + +workqueue = Queue.new +times = {} + +# Create all the thread and have them loop on the work queue +threads = (1..parallel).map{|n| + Thread.new { + loop do + u = workqueue.pop + break if u == :exit + times[u] = get_dates(u, distros, archs) + end + } +} + +# Push all mirrors into the queue +mirrors.each{|u| + workqueue << u +} + +# Get all the threads to exit after all the work is done +parallel.times{|i| + workqueue << :exit +} + +# Wait for the threads to exit +threads.each{|t| + t.join +} + +# Generate output +print_output(distros, archs, mirrors, ref_times, times) + diff --git a/modules/mga-mirrors/templates/cron-mga_mirrors b/modules/mga-mirrors/templates/cron-mga_mirrors index 0f9410a8..96b08bd8 100644 --- a/modules/mga-mirrors/templates/cron-mga_mirrors +++ b/modules/mga-mirrors/templates/cron-mga_mirrors @@ -1,2 +1,3 @@ MAILTO=root 0 * * * * apache /usr/bin/check_mirror > /dev/null 2>&1 +*/20 * * * * nobody /usr/local/bin/check_mirrors --output /var/www/mirrors/status.html.tmp && mv -f /var/www/mirrors/status.html.tmp /var/www/mirrors/status.html -- cgit v1.2.1