summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPascal Terjan <pterjan@gmail.com>2014-11-10 00:35:14 (GMT)
committerPascal Terjan <pterjan@gmail.com>2014-11-10 00:35:14 (GMT)
commit46222ceb3586394df7e9436ee68bfcb4c95abd63 (patch)
treeba6a424291c27e92912306b4148b6d74df399c02
Add some of the ugly scripts used for autobuildHEADmaster
-rwxr-xr-xautobuild.rb172
-rwxr-xr-xcleanlogs.sh5
-rwxr-xr-xgetfromshards.sh18
-rwxr-xr-xrebuild.sh21
4 files changed, 216 insertions, 0 deletions
diff --git a/autobuild.rb b/autobuild.rb
new file mode 100755
index 0000000..599b03b
--- /dev/null
+++ b/autobuild.rb
@@ -0,0 +1,172 @@
+#!/usr/bin/ruby
+
+require 'fileutils'
+require 'rubygems'
+require 'sqlite3'
+
+def create_tables(db)
+ #db.execute "CREATE TABLE IF NOT EXISTS Packages(Id INTEGER PRIMARY KEY, Name TEXT, Version TEXT, Release TEXT, UNIQUE (Name, Version, Release))"
+ db.execute "CREATE TABLE IF NOT EXISTS Packages(Id INTEGER PRIMARY KEY, Name TEXT, UNIQUE (Name))"
+ db.execute "CREATE TABLE IF NOT EXISTS Runs(Id INTEGER PRIMARY KEY, Start INTEGER, End INTEGER, UNIQUE(Start))"
+ db.execute "CREATE TABLE IF NOT EXISTS ResultValues(Id INTEGER PRIMARY KEY, Name TEXT, UNIQUE(Name))"
+ db.execute "CREATE TABLE IF NOT EXISTS Results(Id INTEGER PRIMARY KEY, Package INTEGER, Run INTEGER, Result INTEGER,
+ FOREIGN KEY(Package) REFERENCES Packages(Id),
+ FOREIGN KEY(Run) REFERENCES Run(Id),
+ FOREIGN KEY(Result) REFERENCES ResultValues(Id),
+ UNIQUE(Package, Run, Result))"
+ db.execute "CREATE TABLE IF NOT EXISTS Diffs(Run INTEGER PRIMARY KEY, New INTEGER, Fixed INTEGER, NotFixed INTEGER)"
+end
+
+def get_or_add_resultvalue(db, result)
+ db.execute "INSERT OR IGNORE INTO ResultValues(Name) VALUES('#{result}')"
+ return db.execute("SELECT Id FROM ResultValues WHERE Name = '#{result}'")[0][0]
+end
+
+def get_or_add_package(db, name, version, release)
+ #db.execute "INSERT OR IGNORE INTO Packages(Name, Version, Release) VALUES('#{name}','#{version}','#{release}')"
+ #return db.execute("SELECT Id FROM Packages WHERE Name = '#{name}' AND Version = '#{version}' AND Release = '#{release}'")[0][0]
+ db.execute "INSERT OR IGNORE INTO Packages(Name) VALUES('#{name}')"
+ return db.execute("SELECT Id FROM Packages WHERE Name = '#{name}'")[0][0]
+end
+
+def insert_run(db, status_file)
+ t_start = Date.parse(File.basename(File.dirname(status_file))).strftime('%s')
+ puts "Inserting data for run #{t_start} (#{status_file})"
+ t_end = File.mtime(status_file).to_i
+ db.execute "INSERT INTO Runs(Start, End) VALUES(#{t_start}, #{t_end})"
+ run_id = db.last_insert_row_id
+ File.open(status_file, 'r') {|f|
+ db.transaction
+ f.each_line{|l|
+ if l !~ /^(.*)-([^-]*)-([^-]*).src.rpm: (.*)$/ then
+ puts l
+ next
+ end
+ name = $1
+ version = $2
+ release = $3
+ result = $4
+ if result == 'rejected' then
+ next
+ end
+ result_id = get_or_add_resultvalue(db, result)
+ package_id = get_or_add_package(db, name, version, release)
+ puts name
+ db.execute "INSERT INTO Results(Package, Run, Result) VALUES(#{package_id}, #{run_id}, #{result_id})"
+ }
+ db.commit
+ }
+end
+
+def drop_run(db, r)
+ t_start = Date.parse(r).strftime('%s')
+ run_id = db.execute("SELECT Id FROM Runs WHERE Start='#{t_start}'")[0][0]
+ puts "Dropping run #{t_start}"
+ db.execute "DELETE FROM Results WHERE Run = #{run_id}"
+ db.execute "DELETE FROM Runs WHERE Id = #{run_id}"
+end
+
+def diff_runs(db, r1, r2)
+ (newly_broken, fixed, still_broken) = (db.execute "SELECT New, Fixed, NotFixed FROM DIffs WHERE Run = #{r2}")[0]
+ if newly_broken.nil? then
+ ok_id = get_or_add_resultvalue(db, 'ok')
+ not_on_this_arch_id = get_or_add_resultvalue(db, 'not_on_this_arch')
+ failure_query = "Result NOT IN (#{ok_id}, #{not_on_this_arch_id})"
+ newly_broken = (db.execute "SELECT count(Id) FROM Results WHERE Run = #{r2} AND #{failure_query} AND Package NOT IN (SELECT Package FROM Results WHERE Run = #{r1} AND #{failure_query})")[0][0]
+ fixed = (db.execute "SELECT count(Results.Id) FROM Results,Packages WHERE Run = #{r2} AND Package = Packages.Id AND (Result = #{ok_id} OR Result = #{not_on_this_arch_id}) AND Name IN (SELECT Name FROM Results,Packages WHERE Run = #{r1} AND #{failure_query} AND Package = Packages.Id)")[0][0]
+ still_broken = (db.execute "SELECT count(Id) FROM Results WHERE Run = #{r2} AND #{failure_query} AND Package IN (SELECT Package FROM Results WHERE Run = #{r1} AND #{failure_query})")[0][0]
+ db.execute "INSERT INTO Diffs(Run, New, Fixed, NotFixed) VALUES(#{r2},#{newly_broken},#{fixed},#{still_broken})"
+ end
+ { 'new' => newly_broken, 'fixed' => fixed, 'not fixed' => still_broken }
+end
+
+def report(db)
+ resultvalues = db.execute("SELECT Name FROM ResultValues").flatten
+ runs = db.execute("SELECT Id, Start FROM Runs ORDER BY Start ASC")
+ results = {}
+ prev_run = 0
+ runs.each{|r|
+ run_id = r[0]
+ results[run_id] = {}
+ data = db.execute "SELECT count(Results.Id), ResultValues.Name FROM Results, ResultValues WHERE Results.Result = ResultValues.Id and Run = #{run_id} GROUP BY ResultValues.Name"
+ total = 0
+ success = 0
+ failure = 0
+ data.each{|d|
+ results[run_id][d[1]] = d[0]
+ total += d[0]
+ if (d[1] == "ok")
+ success += d[0]
+ elsif (d[1] == "build_failure" || d[1] == "missing_dep" || d[1] == "install_deps_failure" || d[1] == "recreate_srpm_failure")
+ failure += d[0]
+ end
+ results[run_id]['success'] = success
+ results[run_id]['failure'] = failure
+ results[run_id]['total'] = total
+ results[run_id]['success_rate'] = (1000*success/(success+failure)).round.to_f/10
+ }
+ results[run_id].merge!(diff_runs(db, prev_run, run_id))
+ prev_run = run_id
+ }
+ results
+end
+
+def export_data(db, target_dir)
+ resultvalues = db.execute("SELECT Name FROM ResultValues").flatten
+ runs = db.execute("SELECT Id, Start FROM Runs ORDER BY Start DESC")
+ results = report(db)
+ File.open(target_dir + "/data.js", 'w') {|df|
+ df.puts "var data = { cols: [ {id: 'A', label: 'Date', type: 'date'},"
+ col = 'A'
+ resultvalues.each{|v| df.puts "{id: '#{col.next!}', label: '#{v}', type: 'number'}," }
+ df.puts "{id: '#{col.next!}', label: 'Total', type: 'number'},"
+ df.puts "{id: '#{col.next!}', label: 'Success Rate', type: 'number'},"
+ df.puts "{id: '#{col.next!}', label: 'Newly Broken', type: 'number'},"
+ df.puts "{id: '#{col.next!}', label: 'Fixed', type: 'number'},"
+ df.puts "{id: '#{col.next!}', label: 'Not Fixed', type: 'number'},"
+ df.puts "],"
+ df.puts "rows: ["
+ runs.each{|r|
+ result = results[r[0]]
+ date_val = "new Date(#{DateTime.strptime(r[1].to_s, '%s').strftime('%Y, %m-1, %d')}, 0, 0, 0)"
+ date_str = DateTime.strptime(r[1].to_s, '%s').strftime('%Y-%m-%d')
+ df.print "{c:[{v: #{date_val}, f: '<a href=\"results.php?run=#{date_str}\">#{date_str}</a>'},"
+ resultvalues.each{|v| df.print "{v: #{result[v] || 0}},"}
+ df.print "{v: #{result['total']}},"
+ df.print "{v: #{result['success_rate']}, f: '#{result['success_rate']}%'},"
+ df.print "{v: #{result['new']}},"
+ df.print "{v: #{result['fixed']}},"
+ df.print "{v: #{result['not fixed']}},"
+ df.puts "]},"
+ }
+ df.puts "]}"
+ }
+ FileUtils.cp("autobuild.db", target_dir)
+end
+
+def finish
+ # get date of finished one
+ # olddate = readlink current latest
+ # change link to point to newdate
+ # cleanlogs.sh olddate newdate
+ # insert newdate
+end
+
+begin
+ db = SQLite3::Database.open "autobuild.db"
+ create_tables(db)
+ if (ARGV[0] == 'insert') then
+ insert_run(db, ARGV[1])
+ elsif (ARGV[0] == 'report')
+ export_data(db, ARGV[1])
+ elsif (ARGV[0] == 'drop')
+ drop_run(db, ARGV[1])
+ end
+rescue SQLite3::Exception => e
+
+ puts "Exception occured"
+ puts e
+
+ensure
+ db.close if db
+end
diff --git a/cleanlogs.sh b/cleanlogs.sh
new file mode 100755
index 0000000..79bfee1
--- /dev/null
+++ b/cleanlogs.sh
@@ -0,0 +1,5 @@
+d1=$1
+d2=$2
+[ -n $1 -a -d ./$1 ] || exit 1
+[ -n $2 -a -d ./$2 ] || exit 1
+for d in $(sort $d1/status.core.log $d2/status.core.log | uniq -d | grep -v this_arch | cut -d: -f1); do rm -rf $d1/$d; ln -s ../$d2/$d $d1/$d; done
diff --git a/getfromshards.sh b/getfromshards.sh
new file mode 100755
index 0000000..a0074f2
--- /dev/null
+++ b/getfromshards.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+date=$1
+ip0=$2
+ip1=$3
+ip2=$4
+
+# TODO: Loop on args instead of hardcoding 3 shards
+
+dir="/var/www/bs/autobuild/cauldron/x86_64/core/${date}"
+
+rsync -a --exclude status.core.log ${ip0}:build/${date}/log/ ${dir}/
+rsync -a --exclude status.core.log ${ip1}:build/${date}/log/ ${dir}/
+rsync -a --exclude status.core.log ${ip2}:build/${date}/log/ ${dir}/
+scp ${ip0}:build/${date}/log/status.core.log ${dir}/status.core.log.0
+scp ${ip1}:build/${date}/log/status.core.log ${dir}/status.core.log.1
+scp ${ip2}:build/${date}/log/status.core.log ${dir}/status.core.log.2
+sort ${dir}/status.core.log.{0,1,2} > ${dir}/status.core.log
diff --git a/rebuild.sh b/rebuild.sh
new file mode 100755
index 0000000..df98c54
--- /dev/null
+++ b/rebuild.sh
@@ -0,0 +1,21 @@
+# Sample script to rebuild the distribution
+
+# mount -t tmpfs none /home/pterjan/build/ -o size=100G,nr_inodes=32M
+stat -f -c %T /home/pterjan/build | grep -q tmpfs || exit 1
+
+IURT="perl -I/home/pterjan/git/iurt/lib /home/pterjan/git/iurt/iurt"
+
+# Rebuild chroot tarball
+$IURT --repository /distrib --discard-packages --chrooted-urpmi http://127.0.0.1/ -r cauldron x86_64 /distrib/cauldron/SRPMS/core/release/iurt-*.src.rpm
+
+dir="$HOME/build/$(date +%Y-%m-%d)"
+[ -d $dir ] && exit 2
+mkdir -p "$dir/log"
+cat /distrib/cauldron/*/VERSION | sort > "$dir/log/VERSION"
+
+if [ "$IURT_NUM_SHARDS" != "" -a "$IURT_SHARD_ID" != "" ]; then
+ shardopts="--num_shards $IURT_NUM_SHARDS --shard_id $IURT_SHARD_ID"
+fi
+nproc=$(/usr/bin/getconf _NPROCESSORS_ONLN)
+
+$IURT --discard-packages --config local_spool "$dir" --repository /distrib --chrooted-urpmi http://127.0.0.1/ --parallel 10 --rpmmacros '%_smp_mflags %(echo %name | grep -q -v "^(auto-multiple-choice|boost|chromium-browser-beta|davfs2|dbus-java|ddclient|dhcp|dmraid|docbook-dtd31-sgml|dosemu|dsniff|eclipse-eclox|eclipse-texlipse|evolution-kolab|fdesktoprecorder|ghostpcl|gmic|inn|kmozillahelper|kscd|kscreen|lincity-ng|lv2|mdadm|modello|monkeystudio|most|motion|munin|mysql-workbench|mythtv|nabi|opencascade|opencpn|openjade|ORBit2|pam|pcb|plotutils|polkit|python-cairo|python-dtopt|python-qwt|qbittorrent|qeven|qterminal|qtiplot|qtwebkit|qupzilla|resolvconf|rosegarden|rt-tests|scidavis|scidvspc|scite|screenfetch|semantik|slrn|spamassassin|spring|springlobby|stardict-quick-eng-fra|sugar-base|syslog-ng|tcptraceroute|thunderbird|tor|util-vserver|waf|wine-gecko|wine-mono|xaos|xmms2|yaflight|evolution-kolab|spring|qtwebkit5|mysql-workbench|xdvik|libvirt-glib)$" && echo -n "-j4 " || echo "-l'$nproc' -j'$nproc'")' '%_install_langs en_GB:en_US:fr:fr_FR' --distro cauldron -a x86_64 --build-all $shardopts -m core > "$dir/iurt.log" 2>&1