aboutsummaryrefslogtreecommitdiffstats
path: root/modules/mga-mirrors/files/check_mirrors_status
blob: 0e23eeca175fa497885a5e88f1378b6d37dc0ea7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
#!/usr/bin/ruby

require 'date'
require 'net/http'
require 'optparse'
require 'thread'
require 'uri'

def get_dates(base, archs_per_distro, optional=true)
  r = {}
  begin
    r['base'] = get_timestamp(base)
  rescue Net::OpenTimeout, Timeout::Error, ArgumentError, NoMethodError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, IOError => e
  end

  archs_per_distro.each{|d, archs|
    r[d] = {}
    archs.each{|a|
      begin
        r[d][a] = get_date(base, d, a)
      rescue Net::OpenTimeout, Timeout::Error, ArgumentError, NoMethodError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, IOError => e
        if !optional then
          STDERR.puts "Failed to fetch #{version_url(base, d, a)}"
          raise
        end
      end
    }
  }
  r
end

def get_mirrors
  # TODO Get it from the DB
  mirrors = []
  fetch_url("http://mirrors.mageia.org/").each_line{|l|
    next unless l =~ /http:.*>http/ 
    url = l.sub(/<a href="(http[^"]*)".*\n/, '\1')
    url += "/" unless url =~ /\/$/
    mirrors << url
  }
  mirrors
end

def fetch_url(url, redirect_limit = 3)
  return if redirect_limit < 0
  if url =~ /^\// then
    open(url){|f|
      return f.read
    }
  else
    uri = URI.parse(url)
    http = Net::HTTP.new(uri.host, uri.port)
    http.open_timeout = 5
    http.read_timeout = 5
    # Ruby 1.8.7 doesn't set a default User-Agent which causes at
    # least one mirror to return 403
    response = http.get(uri.path, {'User-Agent' => 'check_mirrors'})
    case response
    when Net::HTTPSuccess then
      return response.body
    when Net::HTTPRedirection then
      location = response['location']
      # Make location absolute if it was not
      if location =~ /:\/\// then
        fetch_url(location, redirect_limit - 1)
      else
        uri.path = location
        fetch_url(uri.to_s, redirect_limit - 1)
      end
    end
  end
end

def timestamp_url(url)
  "#{url}mageia_timestamp"
end

def get_timestamp(url)
  DateTime.strptime(fetch_url(timestamp_url(url)).to_i.to_s, '%s')
end

def parse_version(version)
  date = version.sub(/.* (........ ..:..)$/, '\1').rstrip
  DateTime.strptime(date, '%Y%m%d %H:%M')
end

def version_url(url, distrib, arch)
  "#{url}distrib/#{distrib}/#{arch}/VERSION"
end

def get_date(url, distrib, arch)
  return parse_version(fetch_url(version_url(url, distrib, arch)))
end

def format_age(ref_time, time)
  return "  <td class='broken'>X</td>" unless time

  diff = time - ref_time
  cls = nil
  if diff == 0 then
    cls = 'ok'
  elsif diff < 0.5 then
    cls = 'almost'
  elsif diff < 2 then
    cls = 'bad'
  end
  if cls == 'ok' then
    return "  <td class='#{cls}'>&nbsp;</td>"
  else
    return "  <td class='#{cls}'>#{time.strftime("%F %R")}</td>"
  end
end

def print_output(archs_per_distro, mirrors, ref_times, times)
  puts "<html>
<style>
td.broken {background-color:#FF0033;}
td.bad {background-color:#FF9933;}
td.almost {background-color:#CCFF66;}
td.ok {background-color:#00FF66;}

td {text-align:center;}
td.name {text-align:left;}

td.sep {width:12px;}

th {background-color:#EEEEEE;}
</style>
<body>"
  puts "Last checked on #{Time.now.strftime("%F %R")}<br/>"
  puts "<table cellpadding='4px'><tr><td class='ok'>Up to date</td><td class='almost'>Less than 12h old</td><td class='bad'>Less than 2 days old</td><td class='broken'>Old or broken</td></tr></table>"
  puts "<table><thead>"
  puts "<tr><td/>"
  puts "<td/><th>Base directory</th>"
  archs_per_distro.each{|d, archs|
	 nb_arches = archs.size
	 puts "  <td/><th colspan='#{nb_arches}'>#{d}</th>"
  }
  puts "</tr>"
  puts "<tr><td/><td/><td/>"
  archs_per_distro.each{|d, archs|
	  puts "  <td class='sep' />"
	  archs.each{|a| 
		puts "  <th>#{a}</th>"
	  }
  }
  puts "</tr></thead>"
  puts "<tbody>"
  puts "<tr><td class='name'>Reference</td>"
  puts "  <td class='sep' />"
  puts "  <td>#{ref_times['base'].strftime("%F %R")}</td>"
  archs_per_distro.each{|d, archs|
    puts "  <td class='sep' />"
    archs.each{|a|
      puts "  <td>#{ref_times[d][a].strftime("%F %R")}</td>"
    }
  }
  puts "</tr>"

  mirrors.each{|u|
    puts "<tr><td class='name'><a href='#{u}'>#{u}</a></td>"
    puts "  <td class='sep' />"
    puts format_age(ref_times['base'], times[u]['base'])
    archs_per_distro.each{|d, archs|
      puts "  <td class='sep' />"
      archs.each{|a|
        puts format_age(ref_times[d][a], times[u][d][a])
      }
    }
    puts "</tr>"
  }
  puts "</tbody></table>"
  puts "</body></html>"
end



# Defaults
ref = 'http://repository.mageia.org/'
archs_per_distro = {
	'7' => ['i586', 'x86_64', 'armv7hl', 'aarch64'],
	'cauldron' => ['i586', 'x86_64', 'armv7hl', 'aarch64']
}
parallel = 8

OptionParser.new {|opts|
  opts.banner = "Usage: #{$0} [options]"
  opts.on("--repository URL",
          "Reference repository. Default: #{ref}") {
    |url| ref = url
  }
  opts.on("--parallel n", Integer,
	  "Max number of parallel connections. Default: #{parallel}") {
    |n| $parallel = n
  }
  opts.on("--output file",
	  "Write output into given file. Default to STDOUT") {
    |f| $stdout.reopen(f, "w")
  }
}.parse!

# Get dates from the reference repository, and fail if some requested distros 
# or archs are missing
ref_times = get_dates(ref, archs_per_distro, false)

# Get the list of mirror URLs to check
mirrors = get_mirrors

workqueue = Queue.new
times = {}

# Create all the thread and have them loop on the work queue
threads = (1..parallel).map{|n|
  Thread.new {
    loop do
      u = workqueue.pop
      break if u == :exit
      times[u] = get_dates(u, archs_per_distro)
    end
  }
}

# Push all mirrors into the queue
mirrors.each{|u|
  workqueue << u
}

# Get all the threads to exit after all the work is done
parallel.times{|i|
  workqueue << :exit
}

# Wait for the threads to exit
threads.each{|t|
  t.join
}

# Generate output
print_output(archs_per_distro, mirrors, ref_times, times)