From 46222ceb3586394df7e9436ee68bfcb4c95abd63 Mon Sep 17 00:00:00 2001 From: Pascal Terjan Date: Mon, 10 Nov 2014 00:35:14 +0000 Subject: Add some of the ugly scripts used for autobuild --- autobuild.rb | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ cleanlogs.sh | 5 ++ getfromshards.sh | 18 ++++++ rebuild.sh | 21 +++++++ 4 files changed, 216 insertions(+) create mode 100755 autobuild.rb create mode 100755 cleanlogs.sh create mode 100755 getfromshards.sh create mode 100755 rebuild.sh diff --git a/autobuild.rb b/autobuild.rb new file mode 100755 index 0000000..599b03b --- /dev/null +++ b/autobuild.rb @@ -0,0 +1,172 @@ +#!/usr/bin/ruby + +require 'fileutils' +require 'rubygems' +require 'sqlite3' + +def create_tables(db) + #db.execute "CREATE TABLE IF NOT EXISTS Packages(Id INTEGER PRIMARY KEY, Name TEXT, Version TEXT, Release TEXT, UNIQUE (Name, Version, Release))" + db.execute "CREATE TABLE IF NOT EXISTS Packages(Id INTEGER PRIMARY KEY, Name TEXT, UNIQUE (Name))" + db.execute "CREATE TABLE IF NOT EXISTS Runs(Id INTEGER PRIMARY KEY, Start INTEGER, End INTEGER, UNIQUE(Start))" + db.execute "CREATE TABLE IF NOT EXISTS ResultValues(Id INTEGER PRIMARY KEY, Name TEXT, UNIQUE(Name))" + db.execute "CREATE TABLE IF NOT EXISTS Results(Id INTEGER PRIMARY KEY, Package INTEGER, Run INTEGER, Result INTEGER, + FOREIGN KEY(Package) REFERENCES Packages(Id), + FOREIGN KEY(Run) REFERENCES Run(Id), + FOREIGN KEY(Result) REFERENCES ResultValues(Id), + UNIQUE(Package, Run, Result))" + db.execute "CREATE TABLE IF NOT EXISTS Diffs(Run INTEGER PRIMARY KEY, New INTEGER, Fixed INTEGER, NotFixed INTEGER)" +end + +def get_or_add_resultvalue(db, result) + db.execute "INSERT OR IGNORE INTO ResultValues(Name) VALUES('#{result}')" + return db.execute("SELECT Id FROM ResultValues WHERE Name = '#{result}'")[0][0] +end + +def get_or_add_package(db, name, version, release) + #db.execute "INSERT OR IGNORE INTO Packages(Name, Version, Release) VALUES('#{name}','#{version}','#{release}')" + #return db.execute("SELECT Id FROM Packages WHERE Name = '#{name}' AND Version = '#{version}' AND Release = '#{release}'")[0][0] + db.execute "INSERT OR IGNORE INTO Packages(Name) VALUES('#{name}')" + return db.execute("SELECT Id FROM Packages WHERE Name = '#{name}'")[0][0] +end + +def insert_run(db, status_file) + t_start = Date.parse(File.basename(File.dirname(status_file))).strftime('%s') + puts "Inserting data for run #{t_start} (#{status_file})" + t_end = File.mtime(status_file).to_i + db.execute "INSERT INTO Runs(Start, End) VALUES(#{t_start}, #{t_end})" + run_id = db.last_insert_row_id + File.open(status_file, 'r') {|f| + db.transaction + f.each_line{|l| + if l !~ /^(.*)-([^-]*)-([^-]*).src.rpm: (.*)$/ then + puts l + next + end + name = $1 + version = $2 + release = $3 + result = $4 + if result == 'rejected' then + next + end + result_id = get_or_add_resultvalue(db, result) + package_id = get_or_add_package(db, name, version, release) + puts name + db.execute "INSERT INTO Results(Package, Run, Result) VALUES(#{package_id}, #{run_id}, #{result_id})" + } + db.commit + } +end + +def drop_run(db, r) + t_start = Date.parse(r).strftime('%s') + run_id = db.execute("SELECT Id FROM Runs WHERE Start='#{t_start}'")[0][0] + puts "Dropping run #{t_start}" + db.execute "DELETE FROM Results WHERE Run = #{run_id}" + db.execute "DELETE FROM Runs WHERE Id = #{run_id}" +end + +def diff_runs(db, r1, r2) + (newly_broken, fixed, still_broken) = (db.execute "SELECT New, Fixed, NotFixed FROM DIffs WHERE Run = #{r2}")[0] + if newly_broken.nil? then + ok_id = get_or_add_resultvalue(db, 'ok') + not_on_this_arch_id = get_or_add_resultvalue(db, 'not_on_this_arch') + failure_query = "Result NOT IN (#{ok_id}, #{not_on_this_arch_id})" + newly_broken = (db.execute "SELECT count(Id) FROM Results WHERE Run = #{r2} AND #{failure_query} AND Package NOT IN (SELECT Package FROM Results WHERE Run = #{r1} AND #{failure_query})")[0][0] + fixed = (db.execute "SELECT count(Results.Id) FROM Results,Packages WHERE Run = #{r2} AND Package = Packages.Id AND (Result = #{ok_id} OR Result = #{not_on_this_arch_id}) AND Name IN (SELECT Name FROM Results,Packages WHERE Run = #{r1} AND #{failure_query} AND Package = Packages.Id)")[0][0] + still_broken = (db.execute "SELECT count(Id) FROM Results WHERE Run = #{r2} AND #{failure_query} AND Package IN (SELECT Package FROM Results WHERE Run = #{r1} AND #{failure_query})")[0][0] + db.execute "INSERT INTO Diffs(Run, New, Fixed, NotFixed) VALUES(#{r2},#{newly_broken},#{fixed},#{still_broken})" + end + { 'new' => newly_broken, 'fixed' => fixed, 'not fixed' => still_broken } +end + +def report(db) + resultvalues = db.execute("SELECT Name FROM ResultValues").flatten + runs = db.execute("SELECT Id, Start FROM Runs ORDER BY Start ASC") + results = {} + prev_run = 0 + runs.each{|r| + run_id = r[0] + results[run_id] = {} + data = db.execute "SELECT count(Results.Id), ResultValues.Name FROM Results, ResultValues WHERE Results.Result = ResultValues.Id and Run = #{run_id} GROUP BY ResultValues.Name" + total = 0 + success = 0 + failure = 0 + data.each{|d| + results[run_id][d[1]] = d[0] + total += d[0] + if (d[1] == "ok") + success += d[0] + elsif (d[1] == "build_failure" || d[1] == "missing_dep" || d[1] == "install_deps_failure" || d[1] == "recreate_srpm_failure") + failure += d[0] + end + results[run_id]['success'] = success + results[run_id]['failure'] = failure + results[run_id]['total'] = total + results[run_id]['success_rate'] = (1000*success/(success+failure)).round.to_f/10 + } + results[run_id].merge!(diff_runs(db, prev_run, run_id)) + prev_run = run_id + } + results +end + +def export_data(db, target_dir) + resultvalues = db.execute("SELECT Name FROM ResultValues").flatten + runs = db.execute("SELECT Id, Start FROM Runs ORDER BY Start DESC") + results = report(db) + File.open(target_dir + "/data.js", 'w') {|df| + df.puts "var data = { cols: [ {id: 'A', label: 'Date', type: 'date'}," + col = 'A' + resultvalues.each{|v| df.puts "{id: '#{col.next!}', label: '#{v}', type: 'number'}," } + df.puts "{id: '#{col.next!}', label: 'Total', type: 'number'}," + df.puts "{id: '#{col.next!}', label: 'Success Rate', type: 'number'}," + df.puts "{id: '#{col.next!}', label: 'Newly Broken', type: 'number'}," + df.puts "{id: '#{col.next!}', label: 'Fixed', type: 'number'}," + df.puts "{id: '#{col.next!}', label: 'Not Fixed', type: 'number'}," + df.puts "]," + df.puts "rows: [" + runs.each{|r| + result = results[r[0]] + date_val = "new Date(#{DateTime.strptime(r[1].to_s, '%s').strftime('%Y, %m-1, %d')}, 0, 0, 0)" + date_str = DateTime.strptime(r[1].to_s, '%s').strftime('%Y-%m-%d') + df.print "{c:[{v: #{date_val}, f: '#{date_str}'}," + resultvalues.each{|v| df.print "{v: #{result[v] || 0}},"} + df.print "{v: #{result['total']}}," + df.print "{v: #{result['success_rate']}, f: '#{result['success_rate']}%'}," + df.print "{v: #{result['new']}}," + df.print "{v: #{result['fixed']}}," + df.print "{v: #{result['not fixed']}}," + df.puts "]}," + } + df.puts "]}" + } + FileUtils.cp("autobuild.db", target_dir) +end + +def finish + # get date of finished one + # olddate = readlink current latest + # change link to point to newdate + # cleanlogs.sh olddate newdate + # insert newdate +end + +begin + db = SQLite3::Database.open "autobuild.db" + create_tables(db) + if (ARGV[0] == 'insert') then + insert_run(db, ARGV[1]) + elsif (ARGV[0] == 'report') + export_data(db, ARGV[1]) + elsif (ARGV[0] == 'drop') + drop_run(db, ARGV[1]) + end +rescue SQLite3::Exception => e + + puts "Exception occured" + puts e + +ensure + db.close if db +end diff --git a/cleanlogs.sh b/cleanlogs.sh new file mode 100755 index 0000000..79bfee1 --- /dev/null +++ b/cleanlogs.sh @@ -0,0 +1,5 @@ +d1=$1 +d2=$2 +[ -n $1 -a -d ./$1 ] || exit 1 +[ -n $2 -a -d ./$2 ] || exit 1 +for d in $(sort $d1/status.core.log $d2/status.core.log | uniq -d | grep -v this_arch | cut -d: -f1); do rm -rf $d1/$d; ln -s ../$d2/$d $d1/$d; done diff --git a/getfromshards.sh b/getfromshards.sh new file mode 100755 index 0000000..a0074f2 --- /dev/null +++ b/getfromshards.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +date=$1 +ip0=$2 +ip1=$3 +ip2=$4 + +# TODO: Loop on args instead of hardcoding 3 shards + +dir="/var/www/bs/autobuild/cauldron/x86_64/core/${date}" + +rsync -a --exclude status.core.log ${ip0}:build/${date}/log/ ${dir}/ +rsync -a --exclude status.core.log ${ip1}:build/${date}/log/ ${dir}/ +rsync -a --exclude status.core.log ${ip2}:build/${date}/log/ ${dir}/ +scp ${ip0}:build/${date}/log/status.core.log ${dir}/status.core.log.0 +scp ${ip1}:build/${date}/log/status.core.log ${dir}/status.core.log.1 +scp ${ip2}:build/${date}/log/status.core.log ${dir}/status.core.log.2 +sort ${dir}/status.core.log.{0,1,2} > ${dir}/status.core.log diff --git a/rebuild.sh b/rebuild.sh new file mode 100755 index 0000000..df98c54 --- /dev/null +++ b/rebuild.sh @@ -0,0 +1,21 @@ +# Sample script to rebuild the distribution + +# mount -t tmpfs none /home/pterjan/build/ -o size=100G,nr_inodes=32M +stat -f -c %T /home/pterjan/build | grep -q tmpfs || exit 1 + +IURT="perl -I/home/pterjan/git/iurt/lib /home/pterjan/git/iurt/iurt" + +# Rebuild chroot tarball +$IURT --repository /distrib --discard-packages --chrooted-urpmi http://127.0.0.1/ -r cauldron x86_64 /distrib/cauldron/SRPMS/core/release/iurt-*.src.rpm + +dir="$HOME/build/$(date +%Y-%m-%d)" +[ -d $dir ] && exit 2 +mkdir -p "$dir/log" +cat /distrib/cauldron/*/VERSION | sort > "$dir/log/VERSION" + +if [ "$IURT_NUM_SHARDS" != "" -a "$IURT_SHARD_ID" != "" ]; then + shardopts="--num_shards $IURT_NUM_SHARDS --shard_id $IURT_SHARD_ID" +fi +nproc=$(/usr/bin/getconf _NPROCESSORS_ONLN) + +$IURT --discard-packages --config local_spool "$dir" --repository /distrib --chrooted-urpmi http://127.0.0.1/ --parallel 10 --rpmmacros '%_smp_mflags %(echo %name | grep -q -v "^(auto-multiple-choice|boost|chromium-browser-beta|davfs2|dbus-java|ddclient|dhcp|dmraid|docbook-dtd31-sgml|dosemu|dsniff|eclipse-eclox|eclipse-texlipse|evolution-kolab|fdesktoprecorder|ghostpcl|gmic|inn|kmozillahelper|kscd|kscreen|lincity-ng|lv2|mdadm|modello|monkeystudio|most|motion|munin|mysql-workbench|mythtv|nabi|opencascade|opencpn|openjade|ORBit2|pam|pcb|plotutils|polkit|python-cairo|python-dtopt|python-qwt|qbittorrent|qeven|qterminal|qtiplot|qtwebkit|qupzilla|resolvconf|rosegarden|rt-tests|scidavis|scidvspc|scite|screenfetch|semantik|slrn|spamassassin|spring|springlobby|stardict-quick-eng-fra|sugar-base|syslog-ng|tcptraceroute|thunderbird|tor|util-vserver|waf|wine-gecko|wine-mono|xaos|xmms2|yaflight|evolution-kolab|spring|qtwebkit5|mysql-workbench|xdvik|libvirt-glib)$" && echo -n "-j4 " || echo "-l'$nproc' -j'$nproc'")' '%_install_langs en_GB:en_US:fr:fr_FR' --distro cauldron -a x86_64 --build-all $shardopts -m core > "$dir/iurt.log" 2>&1 -- cgit v1.2.1