diff options
Diffstat (limited to 'modules')
-rwxr-xr-x | modules/buildsystem/templates/cleaner.rb | 220 | ||||
-rw-r--r-- | modules/buildsystem/templates/cleaner_test.rb | 46 |
2 files changed, 174 insertions, 92 deletions
diff --git a/modules/buildsystem/templates/cleaner.rb b/modules/buildsystem/templates/cleaner.rb index e92c948b..6356de23 100755 --- a/modules/buildsystem/templates/cleaner.rb +++ b/modules/buildsystem/templates/cleaner.rb @@ -1,114 +1,150 @@ #!/usr/bin/ruby def usage - puts "Usage: #{$0} [options]" - puts "Moves obsolete packages" - puts - puts "-h, --help show help" - puts "-m, --media <path> path to the binary media" - puts "-s, --src <path> path to the associated src media" - puts "-d, --destination <path> path to the old packages storage" + puts "Usage: #{$0} [options]" + puts "Moves obsolete packages" + puts + puts "-h, --help show help" + puts "-m, --media <media> which media to clean (default: core/release)" + puts "-p, --base <path> base path to the repository" + puts "-a, --archs <arch1>,<arch2>,... list of architectures to clean" + puts "-d, --destination <path> path to the old packages storage" + puts "-v, --version <version> version to clean (default: cauldron)" end require 'fileutils' require 'getoptlong' require 'readline' -opts = GetoptLong.new( - [ '--help', '-h', GetoptLong::NO_ARGUMENT ], - [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ], - [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ], - [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ], - [ '--bmedia', '-b', GetoptLong::REQUIRED_ARGUMENT ], - [ '--smedia', '-s', GetoptLong::REQUIRED_ARGUMENT ], - [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ], - [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ] -) - -base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib" -archs = [ "i586", "x86_64" ] -media = "core/release" -old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>" -version = "cauldron" - -opts.each do |opt, arg| - case opt - when '--help' - usage - exit 0 - when '--bmedia' - bin_path = arg.split(",") - when '--smedia' - src_path = arg - when '--destination' - old_path = arg - when '--media' - media = arg - when '--archs' - archs = arg.split(",") - when '--base' - base_path = arg - when '--version' - version = arg - end -end +def process + opts = GetoptLong.new( + [ '--help', '-h', GetoptLong::NO_ARGUMENT ], + [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ], + [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ], + [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ], + [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ], + [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ] + ) + + base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib" + archs = [ "i586", "x86_64", "aarch64", "armv7hl" ] + media = "core/release" + old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>" + version = "cauldron" + + opts.each do |opt, arg| + case opt + when '--help' + usage + exit 0 + when '--destination' + old_path = arg + when '--media' + media = arg + when '--archs' + archs = arg.split(",") + when '--base' + base_path = arg + when '--version' + version = arg + end + end + + src_path = "#{base_path}/#{version}/SRPMS/#{media}" -bin_path ||= archs.map{|arch| "#{base_path}/#{version}/#{arch}/media/#{media}" } -src_path ||= "#{base_path}/#{version}/SRPMS/#{media}" -debug_path = bin_path.map{|path| path.sub("/media/", "/media/debug/")} + $used_srcs = {} + $srcs = {} + $srcages = {} -$used_srcs = {} -$srcs = {} + # Get a list of all src.rpm and their build time -# Get a list of all src.rpm + `urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename:%buildtime' "."`.each_line{|l| + l2 = l.split(':') + filename = l2[0] + buildtime = l2[1].to_i + name = filename.sub(/-[^-]*-[^-]*$/, '') + $srcages[name] = [ filename, buildtime ] unless $srcages[name] && buildtime < $srcages[name][1] + $srcs[filename] = true + } + # TODO: Take an upload lock to avoid being confused by emi moving things + archs.each{|arch| + bin_path = "#{base_path}/#{version}/#{arch}/media/#{media}" + debug_path = bin_path.sub("/media/", "/media/debug/") + old_packages = check_binaries($srcs, $srcages, bin_path, $used_srcs) + old_debug_packages = check_binaries($srcs, {}, debug_path, nil) + move_packages(bin_path, old_path, old_packages) + move_packages(debug_path, old_path, old_debug_packages) + } + $used_srcs.keys.each{|s| $srcs.delete(s)} -`urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename' "."`.each_line{|l| - $srcs[l.rstrip] = true -} + move_packages(src_path, old_path, $srcs.keys) + +end + +def move_packages(src, dst, list) + list.reject!{|f| !File.exist?(src + "/" + f)} + return if list.empty? + list.each{|b| + puts b + } + puts "The #{list.length} listed packages will be moved from #{src} to #{dst}." + line = Readline::readline('Are you sure [Yn]? ') + if (line =~ /^y?$/i) + list.each{|s| + oldfile = src + "/" + s + newfile = dst + "/" + s + next unless File.exist?(oldfile) + if (File.exist?(newfile)) + File.unlink(oldfile) + else + FileUtils.mv(oldfile, newfile) + end + } + end +end # For each binary media: # - Check if we have the src.rpm (else the binary package is obsolete) +# * If we don't have the src.rpm, check if we have a newer version +# - If there is a new version: +# * check if this architecture has packages from it to avoid deleting armv7hl packages before the new one get rebuilt +# * check if the new version is old enough to allow rebuilding everything (7d?) # - Mark used src.rpm (if one is never marked, the src.rpm is obsolete) -def move_packages(src, dst, list) - list.reject!{|f| !File.exist?(src + "/" + f)} - return if list.empty? - list.each{|b| - puts b - } - puts "The #{list.length} listed packages will be moved from #{src} to #{dst}." - line = Readline::readline('Are you sure [Yn]? ') - if (line =~ /^y?$/i) - list.each{|s| - oldfile = src + "/" + s - newfile = dst + "/" + s - next unless File.exist?(oldfile) - if (File.exist?(newfile)) - File.unlink(oldfile) - else - FileUtils.mv(oldfile, newfile) - end - } - end +def packages(path) + `urpmf --synthesis "#{path}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l| + yield l + } end -def check_binaries(path_list, old_path, mark_used) - path_list.each{|bm| - old_binaries = [] - `urpmf --synthesis "#{bm}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l| - l2 = l.split(':') - src = l2[0] - filename = l2[1].rstrip - old_binaries << filename unless $srcs[src] - $used_srcs[src] = true if mark_used - } - move_packages(bm, old_path, old_binaries) - } +def check_binaries(srcs, srcages, path, used_srcs) + used_here_srcs = {} + packages(path) {|l| + l2 = l.split(':') + src = l2[0] + filename = l2[1].rstrip + used_srcs[src] = true if used_srcs != nil + used_here_srcs[src] = true unless filename =~ /noarch.rpm$/ + } + old_binaries = [] + packages(path) {|l| + l2 = l.split(':') + src = l2[0] + filename = l2[1].rstrip + if ! srcs[src] then + name = src.sub(/-[^-]*-[^-]*$/, '') + if srcages[name] then + # If the package was updated, only delete old binaries after 7d + next unless srcages[name][1] < Time.now.to_i - 24*60*60*7 + # Do not delete if the new version of the package hasn't been built for this arch yet + next unless used_here_srcs[srcages[name][0]] + end + old_binaries << filename + end + } + old_binaries end -check_binaries(bin_path, old_path, true) -check_binaries(debug_path, old_path, false) - -$used_srcs.keys.each{|s| $srcs.delete(s)} - -move_packages(src_path, old_path, $srcs.keys) +if __FILE__ == $0 then + process +end diff --git a/modules/buildsystem/templates/cleaner_test.rb b/modules/buildsystem/templates/cleaner_test.rb new file mode 100644 index 00000000..85d46a4b --- /dev/null +++ b/modules/buildsystem/templates/cleaner_test.rb @@ -0,0 +1,46 @@ +require 'cleaner' +require "test/unit" + +class TestCleaner < Test::Unit::TestCase + + @pkgs = [] + + def setpackages(pkgs) + @pkgs = pkgs + end + + def packages(path) + @pkgs.map{|p| yield "#{p}\n"} + end + + def test_old + # Package was built on this arch and src.rpm for new version is 15d old + setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm']) + srcages = {} + srcages['foo'] = [ 'foo-43-1.src.rpm', Time.now.to_i - 15*24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal(['foo-42-1.armv7hl.rpm'], check_binaries(srcs, srcages, '', nil)) + end + + def test_recent + # Package was built on this arch but src.rpm for new version is only 1d old + setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm']) + srcages = {} + srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal([], check_binaries(srcs, srcages, '', nil)) + end + + def test_arm_late + # Package was not yet built on this arch + setpackages(['foo-42-1.src.rpm:foo-42-1.armv7hl.rpm']) + srcages = {} + srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal([], check_binaries(srcs, srcages, '', nil)) + end + +end |