1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
#!/usr/bin/ruby
def usage
puts "Usage: #{$0} [options]"
puts "Moves obsolete packages"
puts
puts "-h, --help show this help"
puts "-a, --archs <arch1>,<arch2>,... list of architectures to clean"
puts "-a, --auto do not ask confirmation"
puts "-p, --base <path> base path to the repository"
puts "-m, --media <media1>,<media2>,... list of media to clean (default: core/release,tainted/release,nonfree/release)"
puts "-d, --destination <path> path to the old packages storage"
puts "-v, --version <version> version to clean (default: cauldron)"
end
require 'fileutils'
require 'getoptlong'
require 'readline'
def process
opts = GetoptLong.new(
[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
[ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ],
[ '--auto', '-A', GetoptLong::NO_ARGUMENT ],
[ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ],
[ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ],
[ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ],
[ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ],
[ '--lockfile', '-l', GetoptLong::REQUIRED_ARGUMENT ],
)
base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib"
archs = [ "i586", "x86_64", "aarch64", "armv7hl" ]
medias = ["core/release", "tainted/release", "nonfree/release"]
old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>"
version = "cauldron"
auto = false
lockfile = nil
opts.each do |opt, arg|
case opt
when '--help'
usage
exit 0
when '--destination'
old_path = arg
when '--media'
medias = arg.split(",")
when '--archs'
archs = arg.split(",")
when '--auto'
auto = true
when '--base'
base_path = arg
when '--version'
version = arg
when '--lockfile'
lockfile = arg
end
end
take_upload_lock(lockfile) if lockfile
medias.each{|media|
src_path = "#{base_path}/#{version}/SRPMS/#{media}"
$used_srcs = {}
$srcs = {}
$srcages = {}
$noarch = {}
# Get a list of all src.rpm and their build time
`urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename:%buildtime:buildarchs' "."`.each_line{|l|
l2 = l.split(':')
filename = l2[0]
buildtime = l2[1].to_i
buildarch = l2[2].rstrip
name = name_from_filename(filename)
$srcages[name] = [ filename, buildtime ] unless $srcages[name] && buildtime < $srcages[name][1]
$srcs[filename] = true
$noarch[name] = true if buildarch == 'noarch'
}
archs.each{|arch|
bin_path = "#{base_path}/#{version}/#{arch}/media/#{media}"
debug_path = bin_path.sub("/media/", "/media/debug/")
old_packages = check_binaries($srcs, $srcages, bin_path, $used_srcs)
old_debug_packages = check_binaries($srcs, {}, debug_path, nil)
move_packages(bin_path, old_path, old_packages, auto)
move_packages(debug_path, old_path, old_debug_packages, auto)
}
$used_srcs.keys.each{|s| $srcs.delete(s)}
move_packages(src_path, old_path, $srcs.keys, auto)
}
end
def take_upload_lock(path)
start_time = Time.new
has_lock = false
at_exit {
if File.exists?(path)
if File.readlines(path)[0].to_i == Process.pid
File.delete(path)
end
end
}
until has_lock
while File.exists?(path)
puts "Upload in progress..."
if Time.new - start_time > 2*3600.0
puts "Could not aquire upload lock for more than 2h, giving up"
end
sleep(5)
end
puts "Taking upload lock"
File.write(path, Process.pid)
if File.readlines(path)[0].to_i == Process.pid
puts "Succesfuly took upload lock"
has_lock = true
end
end
end
def move_packages(src, dst, list, auto)
list.reject!{|f| !File.exist?(src + "/" + f)}
return if list.empty?
list.each{|b|
puts b
}
puts "The #{list.length} listed packages will be moved from #{src} to #{dst}."
line = Readline::readline('Are you sure [Yn]? ') unless auto
if auto || line =~ /^y?$/i
list.each{|s|
oldfile = src + "/" + s
newfile = dst + "/" + s
next unless File.exist?(oldfile)
if (File.exist?(newfile))
File.unlink(oldfile)
else
FileUtils.mv(oldfile, newfile)
end
}
end
end
# For each binary media:
# - Check if we have the src.rpm (else the binary package is obsolete)
# * If we don't have the src.rpm, check if we have a newer version
# - If there is a new version:
# * check if this architecture has packages from it to avoid deleting armv7hl packages before the new one get rebuilt
# * check if the new version is old enough to allow rebuilding everything (7d?)
# - Mark used src.rpm (if one is never marked, the src.rpm is obsolete)
def packages(path)
`urpmf --synthesis "#{path}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename:%buildtime' ":"`.each_line{|l|
l2 = l.split(':')
sourcerpm = l2[0]
filename = l2[1]
buildtime = l2[2].to_i
yield(sourcerpm, filename, buildtime)
}
end
def name_from_filename(filename)
filename.sub(/-[^-]*-[^-]*$/, '')
end
def check_binaries(srcs, srcages, path, used_srcs)
used_here_srcs = {}
all_versions = {}
packages(path) {|src, filename, buildtime|
used_srcs[src] = true if used_srcs != nil
if filename =~ /noarch.rpm$/ then
# We need to mark the src.rpm present on this arch only for full noarch packages
used_here_srcs[src] = true if $noarch[name_from_filename(filename)]
else
used_here_srcs[src] = true
end
name = name_from_filename(filename)
if all_versions[name] then
all_versions[name] << src
else
all_versions[name] = [src]
end
}
old_binaries = []
packages(path) {|src, filename, buildtime|
if ! srcs[src] then
srcname = name_from_filename(src)
if srcages[srcname] then
# The src.rpm is gone but there is a different version of it
# Only delete old binaries after 7d or if there is a new version
name = name_from_filename(filename)
next unless (srcages[srcname][1] < Time.now.to_i - 24*60*60*7 || all_versions[name].include?(src))
# Do not delete newer binaries, upload may be in progress
next unless buildtime < srcages[srcname][1]
# Do not delete if the new version of the package hasn't been built for this arch yet
next unless used_here_srcs[srcages[srcname][0]]
end
old_binaries << filename
end
}
old_binaries
end
if __FILE__ == $0 then
process
end
|
