From 7f5387075ad6711c2c4e2b70f1408867236d0ec3 Mon Sep 17 00:00:00 2001 From: Pascal Terjan Date: Wed, 23 Oct 2019 21:46:23 +0000 Subject: Make the cleanup script smarter Also, refactor it and add tests Now subpackages of a package not yet rebuilt on an architecture do not get clean on that architecture (even the noarch ones from that package) and packages get deleted only if the src package is totally gone or if the new version of it is at least 7 days old. --- modules/buildsystem/templates/cleaner.rb | 220 +++++++++++++++----------- modules/buildsystem/templates/cleaner_test.rb | 46 ++++++ 2 files changed, 174 insertions(+), 92 deletions(-) create mode 100644 modules/buildsystem/templates/cleaner_test.rb (limited to 'modules') diff --git a/modules/buildsystem/templates/cleaner.rb b/modules/buildsystem/templates/cleaner.rb index e92c948b..6356de23 100755 --- a/modules/buildsystem/templates/cleaner.rb +++ b/modules/buildsystem/templates/cleaner.rb @@ -1,114 +1,150 @@ #!/usr/bin/ruby def usage - puts "Usage: #{$0} [options]" - puts "Moves obsolete packages" - puts - puts "-h, --help show help" - puts "-m, --media path to the binary media" - puts "-s, --src path to the associated src media" - puts "-d, --destination path to the old packages storage" + puts "Usage: #{$0} [options]" + puts "Moves obsolete packages" + puts + puts "-h, --help show help" + puts "-m, --media which media to clean (default: core/release)" + puts "-p, --base base path to the repository" + puts "-a, --archs ,,... list of architectures to clean" + puts "-d, --destination path to the old packages storage" + puts "-v, --version version to clean (default: cauldron)" end require 'fileutils' require 'getoptlong' require 'readline' -opts = GetoptLong.new( - [ '--help', '-h', GetoptLong::NO_ARGUMENT ], - [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ], - [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ], - [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ], - [ '--bmedia', '-b', GetoptLong::REQUIRED_ARGUMENT ], - [ '--smedia', '-s', GetoptLong::REQUIRED_ARGUMENT ], - [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ], - [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ] -) - -base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib" -archs = [ "i586", "x86_64" ] -media = "core/release" -old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>" -version = "cauldron" - -opts.each do |opt, arg| - case opt - when '--help' - usage - exit 0 - when '--bmedia' - bin_path = arg.split(",") - when '--smedia' - src_path = arg - when '--destination' - old_path = arg - when '--media' - media = arg - when '--archs' - archs = arg.split(",") - when '--base' - base_path = arg - when '--version' - version = arg - end -end +def process + opts = GetoptLong.new( + [ '--help', '-h', GetoptLong::NO_ARGUMENT ], + [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ], + [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ], + [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ], + [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ], + [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ] + ) + + base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib" + archs = [ "i586", "x86_64", "aarch64", "armv7hl" ] + media = "core/release" + old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>" + version = "cauldron" + + opts.each do |opt, arg| + case opt + when '--help' + usage + exit 0 + when '--destination' + old_path = arg + when '--media' + media = arg + when '--archs' + archs = arg.split(",") + when '--base' + base_path = arg + when '--version' + version = arg + end + end + + src_path = "#{base_path}/#{version}/SRPMS/#{media}" -bin_path ||= archs.map{|arch| "#{base_path}/#{version}/#{arch}/media/#{media}" } -src_path ||= "#{base_path}/#{version}/SRPMS/#{media}" -debug_path = bin_path.map{|path| path.sub("/media/", "/media/debug/")} + $used_srcs = {} + $srcs = {} + $srcages = {} -$used_srcs = {} -$srcs = {} + # Get a list of all src.rpm and their build time -# Get a list of all src.rpm + `urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename:%buildtime' "."`.each_line{|l| + l2 = l.split(':') + filename = l2[0] + buildtime = l2[1].to_i + name = filename.sub(/-[^-]*-[^-]*$/, '') + $srcages[name] = [ filename, buildtime ] unless $srcages[name] && buildtime < $srcages[name][1] + $srcs[filename] = true + } + # TODO: Take an upload lock to avoid being confused by emi moving things + archs.each{|arch| + bin_path = "#{base_path}/#{version}/#{arch}/media/#{media}" + debug_path = bin_path.sub("/media/", "/media/debug/") + old_packages = check_binaries($srcs, $srcages, bin_path, $used_srcs) + old_debug_packages = check_binaries($srcs, {}, debug_path, nil) + move_packages(bin_path, old_path, old_packages) + move_packages(debug_path, old_path, old_debug_packages) + } + $used_srcs.keys.each{|s| $srcs.delete(s)} -`urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename' "."`.each_line{|l| - $srcs[l.rstrip] = true -} + move_packages(src_path, old_path, $srcs.keys) + +end + +def move_packages(src, dst, list) + list.reject!{|f| !File.exist?(src + "/" + f)} + return if list.empty? + list.each{|b| + puts b + } + puts "The #{list.length} listed packages will be moved from #{src} to #{dst}." + line = Readline::readline('Are you sure [Yn]? ') + if (line =~ /^y?$/i) + list.each{|s| + oldfile = src + "/" + s + newfile = dst + "/" + s + next unless File.exist?(oldfile) + if (File.exist?(newfile)) + File.unlink(oldfile) + else + FileUtils.mv(oldfile, newfile) + end + } + end +end # For each binary media: # - Check if we have the src.rpm (else the binary package is obsolete) +# * If we don't have the src.rpm, check if we have a newer version +# - If there is a new version: +# * check if this architecture has packages from it to avoid deleting armv7hl packages before the new one get rebuilt +# * check if the new version is old enough to allow rebuilding everything (7d?) # - Mark used src.rpm (if one is never marked, the src.rpm is obsolete) -def move_packages(src, dst, list) - list.reject!{|f| !File.exist?(src + "/" + f)} - return if list.empty? - list.each{|b| - puts b - } - puts "The #{list.length} listed packages will be moved from #{src} to #{dst}." - line = Readline::readline('Are you sure [Yn]? ') - if (line =~ /^y?$/i) - list.each{|s| - oldfile = src + "/" + s - newfile = dst + "/" + s - next unless File.exist?(oldfile) - if (File.exist?(newfile)) - File.unlink(oldfile) - else - FileUtils.mv(oldfile, newfile) - end - } - end +def packages(path) + `urpmf --synthesis "#{path}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l| + yield l + } end -def check_binaries(path_list, old_path, mark_used) - path_list.each{|bm| - old_binaries = [] - `urpmf --synthesis "#{bm}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l| - l2 = l.split(':') - src = l2[0] - filename = l2[1].rstrip - old_binaries << filename unless $srcs[src] - $used_srcs[src] = true if mark_used - } - move_packages(bm, old_path, old_binaries) - } +def check_binaries(srcs, srcages, path, used_srcs) + used_here_srcs = {} + packages(path) {|l| + l2 = l.split(':') + src = l2[0] + filename = l2[1].rstrip + used_srcs[src] = true if used_srcs != nil + used_here_srcs[src] = true unless filename =~ /noarch.rpm$/ + } + old_binaries = [] + packages(path) {|l| + l2 = l.split(':') + src = l2[0] + filename = l2[1].rstrip + if ! srcs[src] then + name = src.sub(/-[^-]*-[^-]*$/, '') + if srcages[name] then + # If the package was updated, only delete old binaries after 7d + next unless srcages[name][1] < Time.now.to_i - 24*60*60*7 + # Do not delete if the new version of the package hasn't been built for this arch yet + next unless used_here_srcs[srcages[name][0]] + end + old_binaries << filename + end + } + old_binaries end -check_binaries(bin_path, old_path, true) -check_binaries(debug_path, old_path, false) - -$used_srcs.keys.each{|s| $srcs.delete(s)} - -move_packages(src_path, old_path, $srcs.keys) +if __FILE__ == $0 then + process +end diff --git a/modules/buildsystem/templates/cleaner_test.rb b/modules/buildsystem/templates/cleaner_test.rb new file mode 100644 index 00000000..85d46a4b --- /dev/null +++ b/modules/buildsystem/templates/cleaner_test.rb @@ -0,0 +1,46 @@ +require 'cleaner' +require "test/unit" + +class TestCleaner < Test::Unit::TestCase + + @pkgs = [] + + def setpackages(pkgs) + @pkgs = pkgs + end + + def packages(path) + @pkgs.map{|p| yield "#{p}\n"} + end + + def test_old + # Package was built on this arch and src.rpm for new version is 15d old + setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm']) + srcages = {} + srcages['foo'] = [ 'foo-43-1.src.rpm', Time.now.to_i - 15*24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal(['foo-42-1.armv7hl.rpm'], check_binaries(srcs, srcages, '', nil)) + end + + def test_recent + # Package was built on this arch but src.rpm for new version is only 1d old + setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm']) + srcages = {} + srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal([], check_binaries(srcs, srcages, '', nil)) + end + + def test_arm_late + # Package was not yet built on this arch + setpackages(['foo-42-1.src.rpm:foo-42-1.armv7hl.rpm']) + srcages = {} + srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal([], check_binaries(srcs, srcages, '', nil)) + end + +end -- cgit v1.2.1