Make the cleanup script smarter

Also, refactor it and add tests

Now subpackages of a package not yet rebuilt on an architecture do
not get clean on that architecture (even the noarch ones from that
package) and packages get deleted only if the src package is
totally gone or if the new version of it is at least 7 days old.

2 files changed, 174 insertions, 92 deletions

diff --git a/modules/buildsystem/templates/cleaner.rb b/modules/buildsystem/templates/cleaner.rb
index e92c948b..6356de23 100755
--- a/modules/buildsystem/templates/cleaner.rb
+++ b/modules/buildsystem/templates/cleaner.rb
@@ -1,114 +1,150 @@
 #!/usr/bin/ruby
 
 def usage
-	puts "Usage: #{$0} [options]"
-	puts "Moves obsolete packages"
-	puts
-	puts "-h, --help		show help"
-	puts "-m, --media <path>	path to the binary media"
-	puts "-s, --src <path>	path to the associated src media"
-	puts "-d, --destination <path>	path to the old packages storage"
+  puts "Usage: #{$0} [options]"
+  puts "Moves obsolete packages"
+  puts
+  puts "-h, --help    show help"
+  puts "-m, --media <media>  which media to clean (default: core/release)"
+  puts "-p, --base <path>         base path to the repository"
+  puts "-a, --archs <arch1>,<arch2>,...  list of architectures to clean"
+  puts "-d, --destination <path>  path to the old packages storage"
+  puts "-v, --version <version>   version to clean (default: cauldron)"
 end
 
 require 'fileutils'
 require 'getoptlong'
 require 'readline'
 
-opts = GetoptLong.new(
- [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
- [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--bmedia', '-b', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--smedia', '-s', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ],
- [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ]
-)
-
-base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib"
-archs = [ "i586", "x86_64" ]
-media = "core/release"
-old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>"
-version = "cauldron"
-
-opts.each do |opt, arg|
-	case opt
-		when '--help'
-			usage
-			exit 0
-		when '--bmedia'
-			bin_path = arg.split(",")
-		when '--smedia'
-			src_path = arg
-		when '--destination'
-			old_path = arg
-		when '--media'
-			media = arg
-		when '--archs'
-			archs = arg.split(",")
-		when '--base'
-			base_path = arg
-		when '--version'
-			version = arg
-	end
-end
+def process
+  opts = GetoptLong.new(
+    [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+    [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ]
+  )
+
+  base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib"
+  archs = [ "i586", "x86_64", "aarch64", "armv7hl" ]
+  media = "core/release"
+  old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>"
+  version = "cauldron"
+
+  opts.each do |opt, arg|
+    case opt
+      when '--help'
+        usage
+        exit 0
+      when '--destination'
+        old_path = arg
+      when '--media'
+        media = arg
+      when '--archs'
+        archs = arg.split(",")
+      when '--base'
+        base_path = arg
+      when '--version'
+        version = arg
+    end
+  end
+
+  src_path = "#{base_path}/#{version}/SRPMS/#{media}"
 
-bin_path ||= archs.map{|arch| "#{base_path}/#{version}/#{arch}/media/#{media}" }
-src_path ||= "#{base_path}/#{version}/SRPMS/#{media}"
-debug_path = bin_path.map{|path| path.sub("/media/", "/media/debug/")}
+  $used_srcs = {}
+  $srcs = {}
+  $srcages = {}
 
-$used_srcs = {}
-$srcs = {}
+  # Get a list of all src.rpm and their build time
 
-# Get a list of all src.rpm
+  `urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename:%buildtime' "."`.each_line{|l|
+    l2 = l.split(':')
+    filename = l2[0]
+    buildtime = l2[1].to_i
+    name = filename.sub(/-[^-]*-[^-]*$/, '')
+    $srcages[name] = [ filename, buildtime ] unless $srcages[name] && buildtime < $srcages[name][1]
+    $srcs[filename] = true
+  }
+  # TODO: Take an upload lock to avoid being confused by emi moving things
+  archs.each{|arch|
+    bin_path = "#{base_path}/#{version}/#{arch}/media/#{media}"
+    debug_path = bin_path.sub("/media/", "/media/debug/")
+    old_packages = check_binaries($srcs, $srcages, bin_path, $used_srcs)
+    old_debug_packages = check_binaries($srcs, {}, debug_path, nil)
+    move_packages(bin_path, old_path, old_packages)
+    move_packages(debug_path, old_path, old_debug_packages)
+  }
+  $used_srcs.keys.each{|s| $srcs.delete(s)}
 
-`urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename' "."`.each_line{|l|
-	$srcs[l.rstrip] = true
-}
+  move_packages(src_path, old_path, $srcs.keys)
+
+end
+
+def move_packages(src, dst, list)
+  list.reject!{|f| !File.exist?(src + "/" + f)}
+  return if list.empty?
+  list.each{|b|
+    puts b
+  }
+  puts "The #{list.length} listed packages will be moved from #{src} to #{dst}."
+  line = Readline::readline('Are you sure [Yn]? ')
+  if (line =~ /^y?$/i)
+    list.each{|s|
+      oldfile = src + "/" + s
+      newfile = dst + "/" + s
+      next unless File.exist?(oldfile)
+      if (File.exist?(newfile))
+        File.unlink(oldfile)
+      else
+        FileUtils.mv(oldfile, newfile)
+      end
+    }
+  end
+end
 
 # For each binary media:
 # - Check if we have the src.rpm (else the binary package is obsolete)
+#   * If we don't have the src.rpm, check if we have a newer version
+#     - If there is a new version:
+#       * check if this architecture has packages from it to avoid deleting armv7hl packages before the new one get rebuilt
+#       * check if the new version is old enough to allow rebuilding everything (7d?) 
 # - Mark used src.rpm (if one is never marked, the src.rpm is obsolete)
 
-def move_packages(src, dst, list)
-	list.reject!{|f| !File.exist?(src + "/" + f)}
-	return if list.empty?
-	list.each{|b|
-		puts b
-	}
-	puts "The #{list.length} listed packages will be moved from #{src} to #{dst}."
-	line = Readline::readline('Are you sure [Yn]? ')
-	if (line =~ /^y?$/i)
-		list.each{|s|
-			oldfile = src + "/" + s
-			newfile = dst + "/" + s
-			next unless File.exist?(oldfile)
-			if (File.exist?(newfile))
-				File.unlink(oldfile)
-			else
-				FileUtils.mv(oldfile, newfile)
-			end
-		}
-	end
+def packages(path)
+  `urpmf --synthesis "#{path}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l|
+    yield l
+  }
 end
 
-def check_binaries(path_list, old_path, mark_used)
-	path_list.each{|bm|
-		old_binaries = []
-		`urpmf --synthesis "#{bm}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename' ":"`.each_line{|l|
-			l2 = l.split(':')
-			src = l2[0]
-			filename = l2[1].rstrip
-			old_binaries << filename unless $srcs[src]
-			$used_srcs[src] = true if mark_used
-		}
-		move_packages(bm, old_path, old_binaries)
-	}
+def check_binaries(srcs, srcages, path, used_srcs)
+  used_here_srcs = {}
+  packages(path) {|l|
+    l2 = l.split(':')
+    src = l2[0]
+    filename = l2[1].rstrip
+    used_srcs[src] = true if used_srcs != nil
+    used_here_srcs[src] = true unless filename =~ /noarch.rpm$/
+  }
+  old_binaries = []
+  packages(path) {|l|
+    l2 = l.split(':')
+    src = l2[0]
+    filename = l2[1].rstrip
+    if ! srcs[src] then
+      name = src.sub(/-[^-]*-[^-]*$/, '')
+      if srcages[name] then
+        # If the package was updated, only delete old binaries after 7d
+        next unless srcages[name][1] < Time.now.to_i - 24*60*60*7
+        # Do not delete if the new version of the package hasn't been built for this arch yet
+        next unless used_here_srcs[srcages[name][0]]
+      end
+      old_binaries << filename
+    end
+  }
+  old_binaries
 end
 
-check_binaries(bin_path, old_path, true)
-check_binaries(debug_path, old_path, false)
-
-$used_srcs.keys.each{|s| $srcs.delete(s)}
-
-move_packages(src_path, old_path, $srcs.keys)
+if __FILE__ == $0 then
+  process
+end
diff --git a/modules/buildsystem/templates/cleaner_test.rb b/modules/buildsystem/templates/cleaner_test.rb
new file mode 100644
index 00000000..85d46a4b
--- /dev/null
+++ b/modules/buildsystem/templates/cleaner_test.rb
@@ -0,0 +1,46 @@
+require 'cleaner'
+require "test/unit"
+ 
+class TestCleaner < Test::Unit::TestCase
+
+  @pkgs = []
+
+  def setpackages(pkgs)
+    @pkgs = pkgs
+  end
+
+  def packages(path)
+    @pkgs.map{|p| yield "#{p}\n"}
+  end
+
+  def test_old
+    # Package was built on this arch and src.rpm for new version is 15d old
+    setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm'])
+    srcages = {}
+    srcages['foo'] = [ 'foo-43-1.src.rpm', Time.now.to_i - 15*24*3600 ]
+    srcs = {}
+    srcs['foo-43-1.src.rpm'] = true
+    assert_equal(['foo-42-1.armv7hl.rpm'], check_binaries(srcs, srcages, '', nil))
+  end
+
+  def test_recent
+    # Package was built on this arch but src.rpm for new version is only 1d old
+    setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm'])
+    srcages = {}
+    srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ]
+    srcs = {}
+    srcs['foo-43-1.src.rpm'] = true
+    assert_equal([], check_binaries(srcs, srcages, '', nil))
+  end
+
+  def test_arm_late
+    # Package was not yet built on this arch
+    setpackages(['foo-42-1.src.rpm:foo-42-1.armv7hl.rpm'])
+    srcages = {}
+    srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ]
+    srcs = {}
+    srcs['foo-43-1.src.rpm'] = true
+    assert_equal([], check_binaries(srcs, srcages, '', nil))
+  end
+ 
+end