diff options
Diffstat (limited to 'modules/buildsystem/templates/cleaner.rb')
| -rwxr-xr-x | modules/buildsystem/templates/cleaner.rb | 301 |
1 files changed, 211 insertions, 90 deletions
diff --git a/modules/buildsystem/templates/cleaner.rb b/modules/buildsystem/templates/cleaner.rb index 338d66ba..fa0d08ca 100755 --- a/modules/buildsystem/templates/cleaner.rb +++ b/modules/buildsystem/templates/cleaner.rb @@ -1,114 +1,235 @@ #!/usr/bin/ruby def usage - puts "Usage: #{$0} [options]" - puts "Moves obsolete packages" - puts - puts "-h, --help show help" - puts "-m, --media <path> path to the binary media" - puts "-s, --src <path> path to the associated src media" - puts "-d, --destination <path> path to the old packages storage" + puts "Usage: #{$0} [options]" + puts "Moves obsolete packages" + puts + puts "-h, --help show this help" + puts "-a, --archs <arch1>,<arch2>,... list of architectures to clean" + puts "-a, --auto do not ask confirmation" + puts "-p, --base <path> base path to the repository" + puts "-m, --media <media1>,<media2>,... list of media to clean (default: core/release,tainted/release,nonfree/release)" + puts "-d, --destination <path> path to the old packages storage" + puts "-v, --version <version> version to clean (default: cauldron)" end require 'fileutils' require 'getoptlong' require 'readline' -opts = GetoptLong.new( - [ '--help', '-h', GetoptLong::NO_ARGUMENT ], - [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ], - [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ], - [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ], - [ '--bmedia', '-b', GetoptLong::REQUIRED_ARGUMENT ], - [ '--smedia', '-s', GetoptLong::REQUIRED_ARGUMENT ], - [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ], - [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ] -) - -base_path = "<%= repository_root %>/distrib" -archs = [ "i586", "x86_64" ] -media = "core/release" -old_path = "<%= packages_archivedir %>" -version = "cauldron" - -opts.each do |opt, arg| - case opt - when '--help' - usage - exit 0 - when '--bmedia' - bin_path = arg.split(",") - when '--smedia' - src_path = arg - when '--destination' - old_path = arg - when '--media' - media = arg - when '--archs' - archs = arg.split(",") - when '--base' - base_path = arg - when '--version' - version = arg - end -end +def process + opts = GetoptLong.new( + [ '--help', '-h', GetoptLong::NO_ARGUMENT ], + [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ], + [ '--auto', '-A', GetoptLong::NO_ARGUMENT ], + [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ], + [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ], + [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ], + [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ], + [ '--lockfile', '-l', GetoptLong::REQUIRED_ARGUMENT ], + ) + + base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib" + archs = [ "x86_64", "i686", "aarch64", "armv7hl" ] + medias = ["core/release", "tainted/release", "nonfree/release"] + old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>" + version = "cauldron" + auto = false + lockfile = nil + + opts.each do |opt, arg| + case opt + when '--help' + usage + exit 0 + when '--destination' + old_path = arg + when '--media' + medias = arg.split(",") + when '--archs' + archs = arg.split(",") + when '--auto' + auto = true + when '--base' + base_path = arg + when '--version' + version = arg + when '--lockfile' + lockfile = arg + end + end + + take_upload_lock(lockfile) if lockfile -bin_path ||= archs.map{|arch| "#{base_path}/#{version}/#{arch}/media/#{media}" } -src_path ||= "#{base_path}/#{version}/SRPMS/#{media}" -debug_path = bin_path.map{|path| path.sub("/media/", "/media/debug/")} + medias.each{|media| + src_path = "#{base_path}/#{version}/SRPMS/#{media}" -$used_srcs = {} -$srcs = {} + $used_srcs = {} + $old_srcs = {} + $srcs = {} + $srcages = {} + $noarch = {} -# Get a list of all src.rpm + # Get a list of all src.rpm and their build time + `urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename:%buildtime:%buildarchs' "."`.each_line{|l| + l2 = l.split(':') + filename = l2[0] + buildtime = l2[1].to_i + buildarch = l2[2].rstrip + name = name_from_filename(filename) + if $srcages[name] then + if buildtime < $srcages[name][1] then + # This src.rpm is older, ignore it and store it in the list to be deleted + $old_srcs[filename] = true + next + else + # This src.rpm has an older version, ignore that version and store it in the list to be deleted + old_filename = $srcages[name][0] + $old_srcs[old_filename] = true + $srcs.delete(old_filename) + end + end + $srcages[name] = [ filename, buildtime ] + $srcs[filename] = true + $noarch[name] = true if buildarch == 'noarch' + } + archs.each{|arch| + bin_path = "#{base_path}/#{version}/#{arch}/media/#{media}" + debug_path = bin_path.sub("/media/", "/media/debug/") + old_packages = check_binaries(arch, $srcs, $srcages, src_path, bin_path, $used_srcs) + old_debug_packages = check_binaries(arch, $srcs, {}, src_path, debug_path, nil) + move_packages(bin_path, old_path, old_packages, auto) + move_packages(debug_path, old_path, old_debug_packages, auto) + } + $used_srcs.keys.each{|s| $srcs.delete(s)} -`urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename' "."`.each_line{|l| - $srcs[l.rstrip] = true -} + move_packages(src_path, old_path, $srcs.keys + $old_srcs.keys, auto) + } +end + +def take_upload_lock(path) + start_time = Time.new + has_lock = false + at_exit { + if File.exists?(path) + if File.readlines(path)[0].to_i == Process.pid + File.delete(path) + end + end + } + until has_lock + while File.exists?(path) + if Time.new - start_time > 2*3600.0 + puts "Could not acquire upload lock for more than 2h, giving up" + end + sleep(5) + end + File.write(path, Process.pid) + if File.readlines(path)[0].to_i == Process.pid + has_lock = true + end + end +end + +def move_packages(src, dst, list, auto) + list.reject!{|f| !File.exist?(src + "/" + f)} + return if list.empty? + list.each{|b| + puts b + } + puts "The #{list.length} listed packages will be moved from #{src} to #{dst}." + line = Readline::readline('Are you sure [Yn]? ') unless auto + if auto || line =~ /^y?$/i + list.each{|s| + oldfile = src + "/" + s + newfile = dst + "/" + s + next unless File.exist?(oldfile) + if (File.exist?(newfile)) + File.unlink(oldfile) + else + FileUtils.mv(oldfile, newfile) + end + } + end +end # For each binary media: # - Check if we have the src.rpm (else the binary package is obsolete) +# * If we don't have the src.rpm, check if we have a newer version +# - If there is a new version: +# * check if this architecture has packages from it to avoid deleting armv7hl packages before the new one get rebuilt +# * check if the new version is old enough to allow rebuilding everything (7d?) # - Mark used src.rpm (if one is never marked, the src.rpm is obsolete) -def move_packages(src, dst, list) - list.reject!{|f| !File.exist?(src + "/" + f)} - return if list.empty? - list.each{|b| - puts b - } - puts "The #{list.length} listed packages will be moved from #{src} to #{dst}." - line = Readline::readline('Are you sure [Yn]? ') - if (line =~ /^y?$/i) - list.each{|s| - oldfile = src + "/" + s - newfile = dst + "/" + s - next unless File.exist?(oldfile) - if (File.exist?(newfile)) - File.unlink(oldfile) - else - FileUtils.mv(oldfile, newfile) - end - } - end +def packages(path) + `urpmf --synthesis "#{path}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename:%buildtime' ":"`.each_line{|l| + l2 = l.split(':') + sourcerpm = l2[0] + filename = l2[1] + buildtime = l2[2].to_i + yield(sourcerpm, filename, buildtime) + } end -def check_binaries(path_list, old_path, mark_used) - path_list.each{|bm| - old_binaries = [] - `urpmf --synthesis "#{bm}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l| - l2 = l.split(':') - src = l2[0] - filename = l2[1].rstrip - old_binaries << filename unless $srcs[src] - $used_srcs[src] = true if mark_used - } - move_packages(bm, old_path, old_binaries) - } +def name_from_filename(filename) + filename.sub(/-[^-]*-[^-]*$/, '') end -check_binaries(bin_path, old_path, true) -check_binaries(debug_path, old_path, false) +def arch_wanted(src, arch) + exclusive_arch = `rpmquery -p #{src} --qf '[%{EXCLUSIVEARCH} ]'`.rstrip + if exclusive_arch != "" then + if !exclusive_arch.split(/ /).include?(arch) then + return false + end + end + exclude_arch = `rpmquery -p #{src} --qf '[%{EXCLUDEARCH} ]'`.rstrip + if exclude_arch != "" then + if exclude_arch.split(/ /).include?(arch) then + return false + end + end + + return true +end -$used_srcs.keys.each{|s| $srcs.delete(s)} +def check_binaries(arch, srcs, srcages, src_path, path, used_srcs) + used_here_srcs = {} + all_versions = {} + packages(path) {|src, filename, buildtime| + used_srcs[src] = true if used_srcs != nil + if filename =~ /noarch.rpm$/ then + # We need to mark the src.rpm present on this arch only for full noarch packages + used_here_srcs[src] = true if $noarch[name_from_filename(src)] + else + used_here_srcs[src] = true + end + name = name_from_filename(filename) + if all_versions[name] then + all_versions[name] << src + else + all_versions[name] = [src] + end + } + old_binaries = [] + packages(path) {|src, filename, buildtime| + if ! srcs[src] then + srcname = name_from_filename(src) + if srcages[srcname] then + # The src.rpm is gone but there is a different version of it + latestsrc = srcages[srcname][0] + # Only delete old binaries after 7d or if there is a new version + name = name_from_filename(filename) + next unless (srcages[srcname][1] < Time.now.to_i - 24*60*60*7 || all_versions[name].include?(latestsrc)) + # Do not delete if the new version of the package hasn't been built for this arch yet + # but still delete it if it is no longer expected to be built. + next unless (used_here_srcs[latestsrc] || !arch_wanted("#{src_path}/#{latestsrc}", arch)) + end + old_binaries << filename + end + } + old_binaries +end -move_packages(src_path, old_path, $srcs.keys) +if __FILE__ == $0 then + process +end |