aboutsummaryrefslogtreecommitdiffstats
path: root/modules/buildsystem/templates/cleaner.rb
blob: 4047317076947fcfbeae29d9360848a58120e0d6 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
#!/usr/bin/ruby

def usage
  puts "Usage: #{$0} [options]"
  puts "Moves obsolete packages"
  puts
  puts "-h, --help                       show this help"
  puts "-a, --archs <arch1>,<arch2>,...  list of architectures to clean"
  puts "-a, --auto                       do not ask confirmation"
  puts "-p, --base <path>                base path to the repository"
  puts "-m, --media <media1>,<media2>,... list of media to clean (default: core/release,tainted/release,nonfree/release)"
  puts "-d, --destination <path>         path to the old packages storage"
  puts "-v, --version <version>          version to clean (default: cauldron)"
end

require 'fileutils'
require 'getoptlong'
require 'readline'

def process
  opts = GetoptLong.new(
    [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
    [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ],
    [ '--auto', '-A', GetoptLong::NO_ARGUMENT ],
    [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ],
    [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ],
    [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ],
    [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ],
    [ '--lockfile', '-l', GetoptLong::REQUIRED_ARGUMENT ],
  )

  base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib"
  archs = [ "i586", "x86_64", "aarch64", "armv7hl" ]
  medias = ["core/release", "tainted/release", "nonfree/release"]
  old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>"
  version = "cauldron"
  auto = false
  lockfile = nil

  opts.each do |opt, arg|
    case opt
      when '--help'
        usage
        exit 0
      when '--destination'
        old_path = arg
      when '--media'
        medias = arg.split(",")
      when '--archs'
        archs = arg.split(",")
      when '--auto'
        auto = true
      when '--base'
        base_path = arg
      when '--version'
        version = arg
      when '--lockfile'
        lockfile = arg
    end
  end

  take_upload_lock(lockfile) if lockfile

  medias.each{|media|
    src_path = "#{base_path}/#{version}/SRPMS/#{media}"

    $used_srcs = {}
    $srcs = {}
    $srcages = {}
    $noarch = {}

    # Get a list of all src.rpm and their build time
    `urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename:%buildtime:buildarchs' "."`.each_line{|l|
      l2 = l.split(':')
      filename = l2[0]
      buildtime = l2[1].to_i
      buildarch = l2[2].rstrip
      name = name_from_filename(filename)
      $srcages[name] = [ filename, buildtime ] unless $srcages[name] && buildtime < $srcages[name][1]
      $srcs[filename] = true
      $noarch[name] = true if buildarch == 'noarch'
    }
    archs.each{|arch|
      bin_path = "#{base_path}/#{version}/#{arch}/media/#{media}"
      debug_path = bin_path.sub("/media/", "/media/debug/")
      old_packages = check_binaries($srcs, $srcages, bin_path, $used_srcs)
      old_debug_packages = check_binaries($srcs, {}, debug_path, nil)
      move_packages(bin_path, old_path, old_packages, auto)
      move_packages(debug_path, old_path, old_debug_packages, auto)
    }
    $used_srcs.keys.each{|s| $srcs.delete(s)}

    move_packages(src_path, old_path, $srcs.keys, auto)
  }
end

def take_upload_lock(path)
  start_time = Time.new
  has_lock = false
  at_exit {
    if File.exists?(path)
      if File.readlines(path)[0].to_i == Process.pid
        File.delete(path)
      end
    end
  }
  until has_lock
    while File.exists?(path)
      puts "Upload in progress..."
      if Time.new - start_time > 2*3600.0
        puts "Could not aquire upload lock for more than 2h, giving up"
      end
      sleep(5)
    end
    puts "Taking upload lock"
    File.write(path, Process.pid)
    if File.readlines(path)[0].to_i == Process.pid
      puts "Succesfuly took upload lock"
      has_lock = true
    end
  end
end

def move_packages(src, dst, list, auto)
  list.reject!{|f| !File.exist?(src + "/" + f)}
  return if list.empty?
  list.each{|b|
    puts b
  }
  puts "The #{list.length} listed packages will be moved from #{src} to #{dst}."
  line = Readline::readline('Are you sure [Yn]? ') unless auto
  if auto || line =~ /^y?$/i
    list.each{|s|
      oldfile = src + "/" + s
      newfile = dst + "/" + s
      next unless File.exist?(oldfile)
      if (File.exist?(newfile))
        File.unlink(oldfile)
      else
        FileUtils.mv(oldfile, newfile)
      end
    }
  end
end

# For each binary media:
# - Check if we have the src.rpm (else the binary package is obsolete)
#   * If we don't have the src.rpm, check if we have a newer version
#     - If there is a new version:
#       * check if this architecture has packages from it to avoid deleting armv7hl packages before the new one get rebuilt
#       * check if the new version is old enough to allow rebuilding everything (7d?) 
# - Mark used src.rpm (if one is never marked, the src.rpm is obsolete)

def packages(path)
  `urpmf --synthesis "#{path}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename:%buildtime' ":"`.each_line{|l|
    l2 = l.split(':')
    sourcerpm = l2[0]
    filename = l2[1]
    buildtime = l2[2].to_i
    yield(sourcerpm, filename, buildtime)
  }
end

def name_from_filename(filename)
  filename.sub(/-[^-]*-[^-]*$/, '')
end

def check_binaries(srcs, srcages, path, used_srcs)
  used_here_srcs = {}
  all_versions = {}
  packages(path) {|src, filename, buildtime|
    used_srcs[src] = true if used_srcs != nil
    if filename =~ /noarch.rpm$/ then
      # We need to mark the src.rpm present on this arch only for full noarch packages
      used_here_srcs[src] = true if $noarch[name_from_filename(filename)]
    else
      used_here_srcs[src] = true
    end
    name = name_from_filename(filename)
    if all_versions[name] then
      all_versions[name] << src
    else
      all_versions[name] = [src]
    end
  }
  old_binaries = []
  packages(path) {|src, filename, buildtime|
    if ! srcs[src] then
      srcname = name_from_filename(src)
      if srcages[srcname] then
        # The src.rpm is gone but there is a different version of it
        # Only delete old binaries after 7d or if there is a new version
        name = name_from_filename(filename)
        next unless (srcages[srcname][1] < Time.now.to_i - 24*60*60*7 || all_versions[name].include?(src))
        # Do not delete newer binaries, upload may be in progress
        next unless buildtime < srcages[srcname][1]
        # Do not delete if the new version of the package hasn't been built for this arch yet
        next unless used_here_srcs[srcages[srcname][0]]
      end
      old_binaries << filename
    end
  }
  old_binaries
end

if __FILE__ == $0 then
  process
end