Add some of the ugly scripts used for autobuild

4 files changed, 216 insertions, 0 deletions

diff --git a/autobuild.rb b/autobuild.rb
new file mode 100755
index 0000000..599b03b
--- /dev/null
+++ b/autobuild.rb
@@ -0,0 +1,172 @@
+#!/usr/bin/ruby
+
+require 'fileutils'
+require 'rubygems'
+require 'sqlite3'
+
+def create_tables(db)
+    #db.execute "CREATE TABLE IF NOT EXISTS Packages(Id INTEGER PRIMARY KEY, Name TEXT, Version TEXT, Release TEXT, UNIQUE (Name, Version, Release))"
+    db.execute "CREATE TABLE IF NOT EXISTS Packages(Id INTEGER PRIMARY KEY, Name TEXT, UNIQUE (Name))"
+    db.execute "CREATE TABLE IF NOT EXISTS Runs(Id INTEGER PRIMARY KEY, Start INTEGER, End INTEGER, UNIQUE(Start))"
+    db.execute "CREATE TABLE IF NOT EXISTS ResultValues(Id INTEGER PRIMARY KEY, Name TEXT, UNIQUE(Name))"
+    db.execute "CREATE TABLE IF NOT EXISTS Results(Id INTEGER PRIMARY KEY, Package INTEGER, Run INTEGER, Result INTEGER,
+		FOREIGN KEY(Package) REFERENCES Packages(Id),
+		FOREIGN KEY(Run) REFERENCES Run(Id),
+		FOREIGN KEY(Result) REFERENCES ResultValues(Id),
+                UNIQUE(Package, Run, Result))"
+    db.execute "CREATE TABLE IF NOT EXISTS Diffs(Run INTEGER PRIMARY KEY, New INTEGER, Fixed INTEGER, NotFixed INTEGER)"
+end
+
+def get_or_add_resultvalue(db, result)
+    db.execute "INSERT OR IGNORE INTO ResultValues(Name) VALUES('#{result}')"
+    return db.execute("SELECT Id FROM ResultValues WHERE Name = '#{result}'")[0][0]
+end
+
+def get_or_add_package(db, name, version, release)
+    #db.execute "INSERT OR IGNORE INTO Packages(Name, Version, Release) VALUES('#{name}','#{version}','#{release}')"
+    #return db.execute("SELECT Id FROM Packages WHERE Name = '#{name}' AND Version = '#{version}' AND Release = '#{release}'")[0][0]
+    db.execute "INSERT OR IGNORE INTO Packages(Name) VALUES('#{name}')"
+    return db.execute("SELECT Id FROM Packages WHERE Name = '#{name}'")[0][0]
+end
+    
+def insert_run(db, status_file)
+    t_start = Date.parse(File.basename(File.dirname(status_file))).strftime('%s')
+    puts "Inserting data for run #{t_start} (#{status_file})"
+    t_end = File.mtime(status_file).to_i
+    db.execute "INSERT INTO Runs(Start, End) VALUES(#{t_start}, #{t_end})"
+    run_id = db.last_insert_row_id
+    File.open(status_file, 'r') {|f|
+	db.transaction
+        f.each_line{|l|
+            if l !~ /^(.*)-([^-]*)-([^-]*).src.rpm: (.*)$/ then
+                puts l
+                next
+            end
+            name = $1
+            version = $2
+            release = $3
+            result = $4
+	    if result == 'rejected' then
+		    next
+	    end
+	    result_id = get_or_add_resultvalue(db, result)
+	    package_id = get_or_add_package(db, name, version, release)
+	    puts name
+            db.execute "INSERT INTO Results(Package, Run, Result) VALUES(#{package_id}, #{run_id}, #{result_id})"
+        }
+	db.commit
+    }
+end
+
+def drop_run(db, r)
+    t_start = Date.parse(r).strftime('%s')
+    run_id = db.execute("SELECT Id FROM Runs WHERE Start='#{t_start}'")[0][0]
+    puts "Dropping run #{t_start}"
+    db.execute "DELETE FROM Results WHERE Run = #{run_id}"
+    db.execute "DELETE FROM Runs WHERE Id = #{run_id}"
+end
+
+def diff_runs(db, r1, r2)
+    (newly_broken, fixed, still_broken) = (db.execute "SELECT New, Fixed, NotFixed FROM DIffs WHERE Run = #{r2}")[0]
+    if newly_broken.nil? then
+	    ok_id = get_or_add_resultvalue(db, 'ok')
+	    not_on_this_arch_id = get_or_add_resultvalue(db, 'not_on_this_arch')
+	    failure_query = "Result NOT IN (#{ok_id}, #{not_on_this_arch_id})"
+	    newly_broken = (db.execute "SELECT count(Id) FROM Results WHERE Run = #{r2} AND #{failure_query} AND Package NOT IN (SELECT Package FROM Results WHERE Run = #{r1} AND #{failure_query})")[0][0]
+	    fixed = (db.execute "SELECT count(Results.Id) FROM Results,Packages WHERE Run = #{r2} AND Package = Packages.Id AND (Result = #{ok_id} OR Result = #{not_on_this_arch_id}) AND Name IN (SELECT Name FROM Results,Packages WHERE Run = #{r1} AND #{failure_query} AND Package = Packages.Id)")[0][0]
+	    still_broken = (db.execute "SELECT count(Id) FROM Results WHERE Run = #{r2} AND #{failure_query} AND Package IN (SELECT Package FROM Results WHERE Run = #{r1} AND #{failure_query})")[0][0]
+	    db.execute "INSERT INTO Diffs(Run, New, Fixed, NotFixed) VALUES(#{r2},#{newly_broken},#{fixed},#{still_broken})"
+    end
+    { 'new' => newly_broken, 'fixed' => fixed, 'not fixed' => still_broken }
+end
+
+def report(db)
+    resultvalues = db.execute("SELECT Name FROM ResultValues").flatten
+    runs = db.execute("SELECT Id, Start FROM Runs ORDER BY Start ASC")
+    results = {}
+    prev_run = 0
+    runs.each{|r|
+	run_id = r[0]
+	results[run_id] = {}
+        data = db.execute "SELECT count(Results.Id), ResultValues.Name FROM Results, ResultValues WHERE Results.Result = ResultValues.Id and Run = #{run_id} GROUP BY ResultValues.Name"
+	total = 0
+	success = 0
+	failure = 0
+	data.each{|d|
+            results[run_id][d[1]] = d[0]
+	    total += d[0]
+	    if (d[1] == "ok")
+                success += d[0]
+	    elsif (d[1] == "build_failure" || d[1] == "missing_dep" || d[1] == "install_deps_failure" || d[1] == "recreate_srpm_failure")
+		failure += d[0]
+	    end
+	    results[run_id]['success'] = success
+	    results[run_id]['failure'] = failure
+	    results[run_id]['total'] = total
+	    results[run_id]['success_rate'] = (1000*success/(success+failure)).round.to_f/10
+	}
+	results[run_id].merge!(diff_runs(db, prev_run, run_id))
+	prev_run = run_id
+    }
+    results
+end
+
+def export_data(db, target_dir)	
+    resultvalues = db.execute("SELECT Name FROM ResultValues").flatten
+    runs = db.execute("SELECT Id, Start FROM Runs ORDER BY Start DESC")
+    results = report(db)
+    File.open(target_dir + "/data.js", 'w') {|df|    
+    df.puts "var data = { cols: [ {id: 'A', label: 'Date', type: 'date'},"
+    col = 'A'
+    resultvalues.each{|v| df.puts "{id: '#{col.next!}', label: '#{v}', type: 'number'}," }
+    df.puts "{id: '#{col.next!}', label: 'Total', type: 'number'},"
+    df.puts "{id: '#{col.next!}', label: 'Success Rate', type: 'number'},"
+    df.puts "{id: '#{col.next!}', label: 'Newly Broken', type: 'number'},"
+    df.puts "{id: '#{col.next!}', label: 'Fixed', type: 'number'},"
+    df.puts "{id: '#{col.next!}', label: 'Not Fixed', type: 'number'},"
+    df.puts  "],"
+    df.puts  "rows: ["
+    runs.each{|r|
+	result = results[r[0]]
+	date_val = "new Date(#{DateTime.strptime(r[1].to_s, '%s').strftime('%Y, %m-1, %d')}, 0, 0, 0)"
+	date_str = DateTime.strptime(r[1].to_s, '%s').strftime('%Y-%m-%d')
+	df.print "{c:[{v: #{date_val}, f: '<a href=\"results.php?run=#{date_str}\">#{date_str}</a>'},"
+        resultvalues.each{|v| df.print "{v: #{result[v] || 0}},"}
+	df.print "{v: #{result['total']}},"
+	df.print "{v: #{result['success_rate']}, f: '#{result['success_rate']}%'},"
+	df.print "{v: #{result['new']}},"
+	df.print "{v: #{result['fixed']}},"
+	df.print "{v: #{result['not fixed']}},"
+	df.puts "]},"
+    }
+    df.puts "]}"
+    }
+    FileUtils.cp("autobuild.db", target_dir)
+end
+
+def finish
+	# get date of finished one
+	# olddate = readlink current latest
+	# change link to point to newdate
+	# cleanlogs.sh olddate newdate
+	# insert newdate
+end
+
+begin 
+    db = SQLite3::Database.open "autobuild.db"
+    create_tables(db)
+    if (ARGV[0] == 'insert') then
+        insert_run(db, ARGV[1])
+    elsif (ARGV[0] == 'report')
+        export_data(db, ARGV[1])
+    elsif (ARGV[0] == 'drop')
+        drop_run(db, ARGV[1])
+    end
+rescue SQLite3::Exception => e 
+    
+    puts "Exception occured"
+    puts e
+    
+ensure
+    db.close if db
+end
diff --git a/cleanlogs.sh b/cleanlogs.sh
new file mode 100755
index 0000000..79bfee1
--- /dev/null
+++ b/cleanlogs.sh
@@ -0,0 +1,5 @@
+d1=$1
+d2=$2
+[ -n $1 -a -d ./$1 ] || exit 1
+[ -n $2 -a -d ./$2 ] || exit 1
+for d in $(sort $d1/status.core.log $d2/status.core.log | uniq -d | grep -v this_arch | cut -d: -f1); do rm -rf $d1/$d; ln -s ../$d2/$d $d1/$d; done
diff --git a/getfromshards.sh b/getfromshards.sh
new file mode 100755
index 0000000..a0074f2
--- /dev/null
+++ b/getfromshards.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+date=$1
+ip0=$2
+ip1=$3
+ip2=$4
+
+# TODO: Loop on args instead of hardcoding 3 shards
+
+dir="/var/www/bs/autobuild/cauldron/x86_64/core/${date}"
+
+rsync -a --exclude status.core.log ${ip0}:build/${date}/log/ ${dir}/
+rsync -a --exclude status.core.log ${ip1}:build/${date}/log/ ${dir}/
+rsync -a --exclude status.core.log ${ip2}:build/${date}/log/ ${dir}/
+scp ${ip0}:build/${date}/log/status.core.log ${dir}/status.core.log.0
+scp ${ip1}:build/${date}/log/status.core.log ${dir}/status.core.log.1
+scp ${ip2}:build/${date}/log/status.core.log ${dir}/status.core.log.2
+sort ${dir}/status.core.log.{0,1,2} > ${dir}/status.core.log
diff --git a/rebuild.sh b/rebuild.sh
new file mode 100755
index 0000000..df98c54
--- /dev/null
+++ b/rebuild.sh
@@ -0,0 +1,21 @@
+# Sample script to rebuild the distribution
+
+# mount -t tmpfs none /home/pterjan/build/ -o size=100G,nr_inodes=32M
+stat -f -c %T /home/pterjan/build | grep -q tmpfs || exit 1
+
+IURT="perl -I/home/pterjan/git/iurt/lib /home/pterjan/git/iurt/iurt"
+
+# Rebuild chroot tarball
+$IURT --repository /distrib --discard-packages --chrooted-urpmi http://127.0.0.1/ -r cauldron x86_64 /distrib/cauldron/SRPMS/core/release/iurt-*.src.rpm
+
+dir="$HOME/build/$(date +%Y-%m-%d)"
+[ -d $dir ] && exit 2
+mkdir -p "$dir/log"
+cat /distrib/cauldron/*/VERSION | sort > "$dir/log/VERSION"
+
+if [ "$IURT_NUM_SHARDS" != "" -a "$IURT_SHARD_ID" != "" ]; then
+	shardopts="--num_shards $IURT_NUM_SHARDS --shard_id $IURT_SHARD_ID"
+fi
+nproc=$(/usr/bin/getconf _NPROCESSORS_ONLN) 
+
+$IURT --discard-packages --config local_spool "$dir" --repository /distrib --chrooted-urpmi http://127.0.0.1/ --parallel 10 --rpmmacros '%_smp_mflags %(echo %name | grep -q -v "^(auto-multiple-choice|boost|chromium-browser-beta|davfs2|dbus-java|ddclient|dhcp|dmraid|docbook-dtd31-sgml|dosemu|dsniff|eclipse-eclox|eclipse-texlipse|evolution-kolab|fdesktoprecorder|ghostpcl|gmic|inn|kmozillahelper|kscd|kscreen|lincity-ng|lv2|mdadm|modello|monkeystudio|most|motion|munin|mysql-workbench|mythtv|nabi|opencascade|opencpn|openjade|ORBit2|pam|pcb|plotutils|polkit|python-cairo|python-dtopt|python-qwt|qbittorrent|qeven|qterminal|qtiplot|qtwebkit|qupzilla|resolvconf|rosegarden|rt-tests|scidavis|scidvspc|scite|screenfetch|semantik|slrn|spamassassin|spring|springlobby|stardict-quick-eng-fra|sugar-base|syslog-ng|tcptraceroute|thunderbird|tor|util-vserver|waf|wine-gecko|wine-mono|xaos|xmms2|yaflight|evolution-kolab|spring|qtwebkit5|mysql-workbench|xdvik|libvirt-glib)$" && echo -n "-j4 " || echo "-l'$nproc' -j'$nproc'")' '%_install_langs en_GB:en_US:fr:fr_FR' --distro cauldron -a x86_64 --build-all $shardopts -m core > "$dir/iurt.log" 2>&1