aboutsummaryrefslogtreecommitdiffstats
path: root/modules
diff options
context:
space:
mode:
Diffstat (limited to 'modules')
-rwxr-xr-xmodules/buildsystem/templates/cleaner.rb220
-rw-r--r--modules/buildsystem/templates/cleaner_test.rb46
2 files changed, 174 insertions, 92 deletions
diff --git a/modules/buildsystem/templates/cleaner.rb b/modules/buildsystem/templates/cleaner.rb
index e92c948b..6356de23 100755
--- a/modules/buildsystem/templates/cleaner.rb
+++ b/modules/buildsystem/templates/cleaner.rb
@@ -1,114 +1,150 @@
#!/usr/bin/ruby
def usage
- puts "Usage: #{$0} [options]"
- puts "Moves obsolete packages"
- puts
- puts "-h, --help show help"
- puts "-m, --media <path> path to the binary media"
- puts "-s, --src <path> path to the associated src media"
- puts "-d, --destination <path> path to the old packages storage"
+ puts "Usage: #{$0} [options]"
+ puts "Moves obsolete packages"
+ puts
+ puts "-h, --help show help"
+ puts "-m, --media <media> which media to clean (default: core/release)"
+ puts "-p, --base <path> base path to the repository"
+ puts "-a, --archs <arch1>,<arch2>,... list of architectures to clean"
+ puts "-d, --destination <path> path to the old packages storage"
+ puts "-v, --version <version> version to clean (default: cauldron)"
end
require 'fileutils'
require 'getoptlong'
require 'readline'
-opts = GetoptLong.new(
- [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
- [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--bmedia', '-b', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--smedia', '-s', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ]
-)
-
-base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib"
-archs = [ "i586", "x86_64" ]
-media = "core/release"
-old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>"
-version = "cauldron"
-
-opts.each do |opt, arg|
- case opt
- when '--help'
- usage
- exit 0
- when '--bmedia'
- bin_path = arg.split(",")
- when '--smedia'
- src_path = arg
- when '--destination'
- old_path = arg
- when '--media'
- media = arg
- when '--archs'
- archs = arg.split(",")
- when '--base'
- base_path = arg
- when '--version'
- version = arg
- end
-end
+def process
+ opts = GetoptLong.new(
+ [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+ [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ],
+ [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ],
+ [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ],
+ [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ],
+ [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ]
+ )
+
+ base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib"
+ archs = [ "i586", "x86_64", "aarch64", "armv7hl" ]
+ media = "core/release"
+ old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>"
+ version = "cauldron"
+
+ opts.each do |opt, arg|
+ case opt
+ when '--help'
+ usage
+ exit 0
+ when '--destination'
+ old_path = arg
+ when '--media'
+ media = arg
+ when '--archs'
+ archs = arg.split(",")
+ when '--base'
+ base_path = arg
+ when '--version'
+ version = arg
+ end
+ end
+
+ src_path = "#{base_path}/#{version}/SRPMS/#{media}"
-bin_path ||= archs.map{|arch| "#{base_path}/#{version}/#{arch}/media/#{media}" }
-src_path ||= "#{base_path}/#{version}/SRPMS/#{media}"
-debug_path = bin_path.map{|path| path.sub("/media/", "/media/debug/")}
+ $used_srcs = {}
+ $srcs = {}
+ $srcages = {}
-$used_srcs = {}
-$srcs = {}
+ # Get a list of all src.rpm and their build time
-# Get a list of all src.rpm
+ `urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename:%buildtime' "."`.each_line{|l|
+ l2 = l.split(':')
+ filename = l2[0]
+ buildtime = l2[1].to_i
+ name = filename.sub(/-[^-]*-[^-]*$/, '')
+ $srcages[name] = [ filename, buildtime ] unless $srcages[name] && buildtime < $srcages[name][1]
+ $srcs[filename] = true
+ }
+ # TODO: Take an upload lock to avoid being confused by emi moving things
+ archs.each{|arch|
+ bin_path = "#{base_path}/#{version}/#{arch}/media/#{media}"
+ debug_path = bin_path.sub("/media/", "/media/debug/")
+ old_packages = check_binaries($srcs, $srcages, bin_path, $used_srcs)
+ old_debug_packages = check_binaries($srcs, {}, debug_path, nil)
+ move_packages(bin_path, old_path, old_packages)
+ move_packages(debug_path, old_path, old_debug_packages)
+ }
+ $used_srcs.keys.each{|s| $srcs.delete(s)}
-`urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename' "."`.each_line{|l|
- $srcs[l.rstrip] = true
-}
+ move_packages(src_path, old_path, $srcs.keys)
+
+end
+
+def move_packages(src, dst, list)
+ list.reject!{|f| !File.exist?(src + "/" + f)}
+ return if list.empty?
+ list.each{|b|
+ puts b
+ }
+ puts "The #{list.length} listed packages will be moved from #{src} to #{dst}."
+ line = Readline::readline('Are you sure [Yn]? ')
+ if (line =~ /^y?$/i)
+ list.each{|s|
+ oldfile = src + "/" + s
+ newfile = dst + "/" + s
+ next unless File.exist?(oldfile)
+ if (File.exist?(newfile))
+ File.unlink(oldfile)
+ else
+ FileUtils.mv(oldfile, newfile)
+ end
+ }
+ end
+end
# For each binary media:
# - Check if we have the src.rpm (else the binary package is obsolete)
+# * If we don't have the src.rpm, check if we have a newer version
+# - If there is a new version:
+# * check if this architecture has packages from it to avoid deleting armv7hl packages before the new one get rebuilt
+# * check if the new version is old enough to allow rebuilding everything (7d?)
# - Mark used src.rpm (if one is never marked, the src.rpm is obsolete)
-def move_packages(src, dst, list)
- list.reject!{|f| !File.exist?(src + "/" + f)}
- return if list.empty?
- list.each{|b|
- puts b
- }
- puts "The #{list.length} listed packages will be moved from #{src} to #{dst}."
- line = Readline::readline('Are you sure [Yn]? ')
- if (line =~ /^y?$/i)
- list.each{|s|
- oldfile = src + "/" + s
- newfile = dst + "/" + s
- next unless File.exist?(oldfile)
- if (File.exist?(newfile))
- File.unlink(oldfile)
- else
- FileUtils.mv(oldfile, newfile)
- end
- }
- end
+def packages(path)
+ `urpmf --synthesis "#{path}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l|
+ yield l
+ }
end
-def check_binaries(path_list, old_path, mark_used)
- path_list.each{|bm|
- old_binaries = []
- `urpmf --synthesis "#{bm}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l|
- l2 = l.split(':')
- src = l2[0]
- filename = l2[1].rstrip
- old_binaries << filename unless $srcs[src]
- $used_srcs[src] = true if mark_used
- }
- move_packages(bm, old_path, old_binaries)
- }
+def check_binaries(srcs, srcages, path, used_srcs)
+ used_here_srcs = {}
+ packages(path) {|l|
+ l2 = l.split(':')
+ src = l2[0]
+ filename = l2[1].rstrip
+ used_srcs[src] = true if used_srcs != nil
+ used_here_srcs[src] = true unless filename =~ /noarch.rpm$/
+ }
+ old_binaries = []
+ packages(path) {|l|
+ l2 = l.split(':')
+ src = l2[0]
+ filename = l2[1].rstrip
+ if ! srcs[src] then
+ name = src.sub(/-[^-]*-[^-]*$/, '')
+ if srcages[name] then
+ # If the package was updated, only delete old binaries after 7d
+ next unless srcages[name][1] < Time.now.to_i - 24*60*60*7
+ # Do not delete if the new version of the package hasn't been built for this arch yet
+ next unless used_here_srcs[srcages[name][0]]
+ end
+ old_binaries << filename
+ end
+ }
+ old_binaries
end
-check_binaries(bin_path, old_path, true)
-check_binaries(debug_path, old_path, false)
-
-$used_srcs.keys.each{|s| $srcs.delete(s)}
-
-move_packages(src_path, old_path, $srcs.keys)
+if __FILE__ == $0 then
+ process
+end
diff --git a/modules/buildsystem/templates/cleaner_test.rb b/modules/buildsystem/templates/cleaner_test.rb
new file mode 100644
index 00000000..85d46a4b
--- /dev/null
+++ b/modules/buildsystem/templates/cleaner_test.rb
@@ -0,0 +1,46 @@
+require 'cleaner'
+require "test/unit"
+
+class TestCleaner < Test::Unit::TestCase
+
+ @pkgs = []
+
+ def setpackages(pkgs)
+ @pkgs = pkgs
+ end
+
+ def packages(path)
+ @pkgs.map{|p| yield "#{p}\n"}
+ end
+
+ def test_old
+ # Package was built on this arch and src.rpm for new version is 15d old
+ setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm'])
+ srcages = {}
+ srcages['foo'] = [ 'foo-43-1.src.rpm', Time.now.to_i - 15*24*3600 ]
+ srcs = {}
+ srcs['foo-43-1.src.rpm'] = true
+ assert_equal(['foo-42-1.armv7hl.rpm'], check_binaries(srcs, srcages, '', nil))
+ end
+
+ def test_recent
+ # Package was built on this arch but src.rpm for new version is only 1d old
+ setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm'])
+ srcages = {}
+ srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ]
+ srcs = {}
+ srcs['foo-43-1.src.rpm'] = true
+ assert_equal([], check_binaries(srcs, srcages, '', nil))
+ end
+
+ def test_arm_late
+ # Package was not yet built on this arch
+ setpackages(['foo-42-1.src.rpm:foo-42-1.armv7hl.rpm'])
+ srcages = {}
+ srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ]
+ srcs = {}
+ srcs['foo-43-1.src.rpm'] = true
+ assert_equal([], check_binaries(srcs, srcages, '', nil))
+ end
+
+end
generated by cgit v1.2.1 (git 2.21.0) at 2024-09-25 13:42:00 +0000