Add some of the ugly scripts used for autobuild

1 files changed, 172 insertions, 0 deletions

diff --git a/autobuild.rb b/autobuild.rb
new file mode 100755
index 0000000..599b03b
--- /dev/null
+++ b/autobuild.rb
@@ -0,0 +1,172 @@
+#!/usr/bin/ruby
+
+require 'fileutils'
+require 'rubygems'
+require 'sqlite3'
+
+def create_tables(db)
+    #db.execute "CREATE TABLE IF NOT EXISTS Packages(Id INTEGER PRIMARY KEY, Name TEXT, Version TEXT, Release TEXT, UNIQUE (Name, Version, Release))"
+    db.execute "CREATE TABLE IF NOT EXISTS Packages(Id INTEGER PRIMARY KEY, Name TEXT, UNIQUE (Name))"
+    db.execute "CREATE TABLE IF NOT EXISTS Runs(Id INTEGER PRIMARY KEY, Start INTEGER, End INTEGER, UNIQUE(Start))"
+    db.execute "CREATE TABLE IF NOT EXISTS ResultValues(Id INTEGER PRIMARY KEY, Name TEXT, UNIQUE(Name))"
+    db.execute "CREATE TABLE IF NOT EXISTS Results(Id INTEGER PRIMARY KEY, Package INTEGER, Run INTEGER, Result INTEGER,
+		FOREIGN KEY(Package) REFERENCES Packages(Id),
+		FOREIGN KEY(Run) REFERENCES Run(Id),
+		FOREIGN KEY(Result) REFERENCES ResultValues(Id),
+                UNIQUE(Package, Run, Result))"
+    db.execute "CREATE TABLE IF NOT EXISTS Diffs(Run INTEGER PRIMARY KEY, New INTEGER, Fixed INTEGER, NotFixed INTEGER)"
+end
+
+def get_or_add_resultvalue(db, result)
+    db.execute "INSERT OR IGNORE INTO ResultValues(Name) VALUES('#{result}')"
+    return db.execute("SELECT Id FROM ResultValues WHERE Name = '#{result}'")[0][0]
+end
+
+def get_or_add_package(db, name, version, release)
+    #db.execute "INSERT OR IGNORE INTO Packages(Name, Version, Release) VALUES('#{name}','#{version}','#{release}')"
+    #return db.execute("SELECT Id FROM Packages WHERE Name = '#{name}' AND Version = '#{version}' AND Release = '#{release}'")[0][0]
+    db.execute "INSERT OR IGNORE INTO Packages(Name) VALUES('#{name}')"
+    return db.execute("SELECT Id FROM Packages WHERE Name = '#{name}'")[0][0]
+end
+    
+def insert_run(db, status_file)
+    t_start = Date.parse(File.basename(File.dirname(status_file))).strftime('%s')
+    puts "Inserting data for run #{t_start} (#{status_file})"
+    t_end = File.mtime(status_file).to_i
+    db.execute "INSERT INTO Runs(Start, End) VALUES(#{t_start}, #{t_end})"
+    run_id = db.last_insert_row_id
+    File.open(status_file, 'r') {|f|
+	db.transaction
+        f.each_line{|l|
+            if l !~ /^(.*)-([^-]*)-([^-]*).src.rpm: (.*)$/ then
+                puts l
+                next
+            end
+            name = $1
+            version = $2
+            release = $3
+            result = $4
+	    if result == 'rejected' then
+		    next
+	    end
+	    result_id = get_or_add_resultvalue(db, result)
+	    package_id = get_or_add_package(db, name, version, release)
+	    puts name
+            db.execute "INSERT INTO Results(Package, Run, Result) VALUES(#{package_id}, #{run_id}, #{result_id})"
+        }
+	db.commit
+    }
+end
+
+def drop_run(db, r)
+    t_start = Date.parse(r).strftime('%s')
+    run_id = db.execute("SELECT Id FROM Runs WHERE Start='#{t_start}'")[0][0]
+    puts "Dropping run #{t_start}"
+    db.execute "DELETE FROM Results WHERE Run = #{run_id}"
+    db.execute "DELETE FROM Runs WHERE Id = #{run_id}"
+end
+
+def diff_runs(db, r1, r2)
+    (newly_broken, fixed, still_broken) = (db.execute "SELECT New, Fixed, NotFixed FROM DIffs WHERE Run = #{r2}")[0]
+    if newly_broken.nil? then
+	    ok_id = get_or_add_resultvalue(db, 'ok')
+	    not_on_this_arch_id = get_or_add_resultvalue(db, 'not_on_this_arch')
+	    failure_query = "Result NOT IN (#{ok_id}, #{not_on_this_arch_id})"
+	    newly_broken = (db.execute "SELECT count(Id) FROM Results WHERE Run = #{r2} AND #{failure_query} AND Package NOT IN (SELECT Package FROM Results WHERE Run = #{r1} AND #{failure_query})")[0][0]
+	    fixed = (db.execute "SELECT count(Results.Id) FROM Results,Packages WHERE Run = #{r2} AND Package = Packages.Id AND (Result = #{ok_id} OR Result = #{not_on_this_arch_id}) AND Name IN (SELECT Name FROM Results,Packages WHERE Run = #{r1} AND #{failure_query} AND Package = Packages.Id)")[0][0]
+	    still_broken = (db.execute "SELECT count(Id) FROM Results WHERE Run = #{r2} AND #{failure_query} AND Package IN (SELECT Package FROM Results WHERE Run = #{r1} AND #{failure_query})")[0][0]
+	    db.execute "INSERT INTO Diffs(Run, New, Fixed, NotFixed) VALUES(#{r2},#{newly_broken},#{fixed},#{still_broken})"
+    end
+    { 'new' => newly_broken, 'fixed' => fixed, 'not fixed' => still_broken }
+end
+
+def report(db)
+    resultvalues = db.execute("SELECT Name FROM ResultValues").flatten
+    runs = db.execute("SELECT Id, Start FROM Runs ORDER BY Start ASC")
+    results = {}
+    prev_run = 0
+    runs.each{|r|
+	run_id = r[0]
+	results[run_id] = {}
+        data = db.execute "SELECT count(Results.Id), ResultValues.Name FROM Results, ResultValues WHERE Results.Result = ResultValues.Id and Run = #{run_id} GROUP BY ResultValues.Name"
+	total = 0
+	success = 0
+	failure = 0
+	data.each{|d|
+            results[run_id][d[1]] = d[0]
+	    total += d[0]
+	    if (d[1] == "ok")
+                success += d[0]
+	    elsif (d[1] == "build_failure" || d[1] == "missing_dep" || d[1] == "install_deps_failure" || d[1] == "recreate_srpm_failure")
+		failure += d[0]
+	    end
+	    results[run_id]['success'] = success
+	    results[run_id]['failure'] = failure
+	    results[run_id]['total'] = total
+	    results[run_id]['success_rate'] = (1000*success/(success+failure)).round.to_f/10
+	}
+	results[run_id].merge!(diff_runs(db, prev_run, run_id))
+	prev_run = run_id
+    }
+    results
+end
+
+def export_data(db, target_dir)	
+    resultvalues = db.execute("SELECT Name FROM ResultValues").flatten
+    runs = db.execute("SELECT Id, Start FROM Runs ORDER BY Start DESC")
+    results = report(db)
+    File.open(target_dir + "/data.js", 'w') {|df|    
+    df.puts "var data = { cols: [ {id: 'A', label: 'Date', type: 'date'},"
+    col = 'A'
+    resultvalues.each{|v| df.puts "{id: '#{col.next!}', label: '#{v}', type: 'number'}," }
+    df.puts "{id: '#{col.next!}', label: 'Total', type: 'number'},"
+    df.puts "{id: '#{col.next!}', label: 'Success Rate', type: 'number'},"
+    df.puts "{id: '#{col.next!}', label: 'Newly Broken', type: 'number'},"
+    df.puts "{id: '#{col.next!}', label: 'Fixed', type: 'number'},"
+    df.puts "{id: '#{col.next!}', label: 'Not Fixed', type: 'number'},"
+    df.puts  "],"
+    df.puts  "rows: ["
+    runs.each{|r|
+	result = results[r[0]]
+	date_val = "new Date(#{DateTime.strptime(r[1].to_s, '%s').strftime('%Y, %m-1, %d')}, 0, 0, 0)"
+	date_str = DateTime.strptime(r[1].to_s, '%s').strftime('%Y-%m-%d')
+	df.print "{c:[{v: #{date_val}, f: '<a href=\"results.php?run=#{date_str}\">#{date_str}</a>'},"
+        resultvalues.each{|v| df.print "{v: #{result[v] || 0}},"}
+	df.print "{v: #{result['total']}},"
+	df.print "{v: #{result['success_rate']}, f: '#{result['success_rate']}%'},"
+	df.print "{v: #{result['new']}},"
+	df.print "{v: #{result['fixed']}},"
+	df.print "{v: #{result['not fixed']}},"
+	df.puts "]},"
+    }
+    df.puts "]}"
+    }
+    FileUtils.cp("autobuild.db", target_dir)
+end
+
+def finish
+	# get date of finished one
+	# olddate = readlink current latest
+	# change link to point to newdate
+	# cleanlogs.sh olddate newdate
+	# insert newdate
+end
+
+begin 
+    db = SQLite3::Database.open "autobuild.db"
+    create_tables(db)
+    if (ARGV[0] == 'insert') then
+        insert_run(db, ARGV[1])
+    elsif (ARGV[0] == 'report')
+        export_data(db, ARGV[1])
+    elsif (ARGV[0] == 'drop')
+        drop_run(db, ARGV[1])
+    end
+rescue SQLite3::Exception => e 
+    
+    puts "Exception occured"
+    puts e
+    
+ensure
+    db.close if db
+end