From 09d1353e262e15e2a7d683e875c9e091a9fef751 Mon Sep 17 00:00:00 2001
From: Claudio Matsuoka <claudio@mandriva.com>
Date: Tue, 14 Nov 2006 17:24:42 +0000
Subject: Ok, perhaps listing .done files makes it too verbose.

---
 emi | 278 ++++++++++++++++++++++++++++++++++++++++++--------------------------
 1 file changed, 171 insertions(+), 107 deletions(-)

(limited to 'emi')

diff --git a/emi b/emi
index 77774b4..296eb7d 100755
--- a/emi
+++ b/emi
@@ -46,35 +46,65 @@ my $HOME = $ENV{HOME};
 my $configfile = "$HOME/.upload.conf";
 
 my %config_usage = ( 
-    admin => { desc => 'mail address of the bot administrator', default => 'warly@mandriva.com' },
+    admin => {
+	desc => 'mail address of the bot administrator',
+	default => 'warly@mandriva.com'
+    },
     arch => {  
 	desc => "List of arch",
-	default => [ 'i586', 'x86_64', 'ppc' , 'sparc', 'sparc64' ],
-         },
-   default_mail_domain => { desc => "Default mail domain to append", default => 'mandriva.org' },
-    http_queue => { desc => 'Address where log can be consulted', default => 'http://svn.mandriva.com/queue/'},
+	default => [ 'i586', 'x86_64', 'ppc' , 'sparc', 'sparc64' ]
+    },
+    default_mail_domain => {
+	desc => "Default mail domain to append",
+	default => 'mandriva.org'
+    },
+    http_queue => {
+	desc => 'Address where log can be consulted',
+	default => 'http://svn.mandriva.com/queue/'
+    },
     mandatory_arch => {
 	desc => 'List of mandatory architecture to be able to upload',
-	default => [ 'i586', 'x86_64' ],
-      },
-   tmp => { desc => "Temporary directory",  default => "$HOME/tmp/"},
-   root => { desc => 'Architecture root dir', default => "/mnt/BIG/dis/" },
-   cache_home => { desc => 'Where to store the cache files', default => "$HOME/.bugs" },
-   upload_user => { desc => 'User who is uploading packages', default => 'mandrake' },
-   repsys_conf => { desc => 'Path of the repsys.conf file which includes login mail corresponding', default => '/etc/repsys.conf' },
-   queue => { desc => 'root directory of the various upload queues', default => '/home/mandrake/uploads/'},
-   ssh_option => { desc => "SSH options", default => "-o ConectTimeout=20" },
+	default => [ 'i586', 'x86_64' ]
+    },
+    tmp => {
+	desc => "Temporary directory",
+	default => "$HOME/tmp/"
+    },
+    root => {
+	desc => 'Architecture root dir',
+	default => "/mnt/BIG/dis/"
+    },
+    cache_home => {
+	desc => 'Where to store the cache files',
+	default => "$HOME/.bugs"
+    },
+    upload_user => {
+	desc => 'User who is uploading packages',
+	default => 'mandrake'
+    },
+    repsys_conf => {
+	desc => 'Path of repsys.conf file with login mail corresponding',
+	default => '/etc/repsys.conf'
+    },
+    queue => {
+	desc => 'root directory of the various upload queues',
+	default => '/home/mandrake/uploads/'
+    },
+    ssh_option => {
+	desc => "SSH options",
+	default => "-o ConectTimeout=20"
+    },
 );
 
 my $config;
 if (-f $configfile) {
-    $config = do $configfile or die "FATAL $program_name: syntax error in $configfile";
+    $config = do $configfile
+	or die "FATAL $program_name: syntax error in $configfile";
 } else {
     $config = {}
 }
 
 config_usage(\%config_usage, $config) if $run{config_usage};
-
 config_init(\%config_usage, $config, \%run);
 
 my $mail = get_repsys_conf($config->{repsys_conf});
@@ -88,7 +118,6 @@ my $cache =  { arch => {} };
 
 my ($fulldate, $daydate) = get_date();
 $run{daydate} = $daydate;
-print {$run{LOG}} "$program_name\n";
 
 my $todo = "$config->{queue}/todo/";
 my $done = "$config->{queue}/done/";
@@ -96,121 +125,156 @@ my $reject = "$config->{queue}/rejected/";
 
 my %pkg_tree;
 my %excluded;
-check_upload_tree(\%run, $done, 
-    sub {
-	my ($todo, $f, $m, $s, $r) = @_;
-	if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
-	    my ($prefix, $rpm, $arch) = ($1, $2, $3);
-	    print {$run{LOG}} "$program_name: found rpm $rpm ($prefix)\n";
-	    $pkg_tree{$prefix}{path} = "/$f/$m/$s";
-	    $pkg_tree{$prefix}{arch}{$arch} = 1;
-	    $pkg_tree{$prefix}{target} = $f;
-	    $pkg_tree{$prefix}{section} = "$m/$s";
-	    push @{$pkg_tree{$prefix}{srpms}}, $rpm if ($arch eq 'src');
-	    push @{$pkg_tree{$prefix}{rpms}} , $rpm;
-	} elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.done$/) {
-	    my ($prefix, $arch) = ($1, $2);
-	    $cache->{arch}{$prefix}{$arch} = 1
-	} elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.excluded$/) {
-	    my ($prefix, $arch) = ($1, $2);
-	    $cache->{arch}{$prefix}{$arch} = 1;
-	    $excluded{$prefix}{$arch} = 1
-	}
 
-    },
-);
 
-check_upload_tree(\%run, $todo, 
-    sub {
-	my ($todo, $f, $m, $s, $r) = @_;
-	if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
-	    my ($prefix, $rpm, $arch) = ($1, $2, $3);
-	    print {$run{LOG}} "$program_name: found todo rpm $rpm ($prefix)\n";
-	    push @{$pkg_tree{$prefix}{todo}}, $rpm
-	}
-    },
-);
+#
+# Gather data from upload tree
+#
+
+sub done_func {
+    my ($todo, $f, $m, $s, $r) = @_;
+
+    if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
+	my ($prefix, $rpm, $arch) = ($1, $2, $3);
+	plog("found rpm $rpm ($prefix)");
+	$pkg_tree{$prefix}{path} = "/$f/$m/$s";
+	$pkg_tree{$prefix}{arch}{$arch} = 1;
+	$pkg_tree{$prefix}{target} = $f;
+	$pkg_tree{$prefix}{section} = "$m/$s";
+	push @{$pkg_tree{$prefix}{srpms}}, $rpm if ($arch eq 'src');
+	push @{$pkg_tree{$prefix}{rpms}} , $rpm;
+    } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.done$/) {
+	my ($prefix, $arch) = ($1, $2);
+	#plog("found .done ($prefix)");
+	$cache->{arch}{$prefix}{$arch} = 1;
+    } elsif ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*)\.excluded$/) {
+	my ($prefix, $arch) = ($1, $2);
+	plog("found .excluded ($prefix)");
+	$cache->{arch}{$prefix}{$arch} = 1;
+	$excluded{$prefix}{$arch} = 1;
+    }
+}
+
+sub todo_func {
+    my ($todo, $f, $m, $s, $r) = @_;
+
+    if ($r =~ /(\d{14}\.\w+\.\w+\.\d+)_(.*\.([^.]+)\.rpm)$/) {
+	my ($prefix, $rpm, $arch) = ($1, $2, $3);
+	plog("$program_name: found todo rpm $rpm ($prefix)");
+	push @{$pkg_tree{$prefix}{todo}}, $rpm;
+    }
+}
+
+check_upload_tree(\%run, $done, \&done_func, );
+check_upload_tree(\%run, $todo, \&todo_func, );
+
 # Once the tree is checked, ulri can be launched again
 unlink $pidfile;
 
+
+#
+# Decide what should be uploaded
+#
+
 foreach my $prefix (keys %pkg_tree) {
     my $ok = 1;
     my $target = $pkg_tree{$prefix}{target};
     my $path = $pkg_tree{$prefix}{path};
     my $section = $pkg_tree{$prefix}{section};
-    print {$run{LOG}} "$program_name: processing $prefix in $path\n";
+
+    plog("processing $prefix in $path");
     my %missing;
     foreach my $mandatory_arch (@{$config->{mandatory_arch}}, 'src') {
 	$excluded{$prefix}{$mandatory_arch} and next;
-	print {$run{LOG}} "$program_name: checking if mandatory architecture $mandatory_arch is present: ";
+	my $x = "yes";
 	if (!$pkg_tree{$prefix}{arch}{$mandatory_arch} && !$pkg_tree{$prefix}{arch}{noarch}){
 	    if (!$cache->{arch}{$prefix}{$mandatory_arch}) {
 		$missing{$mandatory_arch} = 1;
-		print {$run{LOG}} "no\n";
-		$ok = 0
+		$x = "no";
+		$ok = 0;
 	    } else {
-		print {$run{LOG}} "yes (in cache)\n"
+		$x = "yes (in cache)";
 	    }
-	} else {
-	    print {$run{LOG}} "yes\n"
 	}
+	plog(" mandatory architecture $mandatory_arch present: $x");
+    }
+
+    unless ($ok) {
+	plog(0, "WARNING: mandatory arch", join(' ', keys %missing),
+				"missing, waiting");
+	next;
+    }
+
+    #
+    # All mandatory archs found, upload
+    #
+
+    my @packages;
+    my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+$/;
+
+    foreach my $rpm (@{$pkg_tree{$prefix}{rpms}}) {
+	push @packages, "$done/$path/${prefix}_$rpm"
     }
-    if ($ok) {
-	# upload
-	my @packages;
-	my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+$/;
+
+    $user ||= $config->{upload_user};
+
+    my $command = "/usr/bin/sudo -H -u $config->{upload_user} /usr/bin/perl -I/usr/local/lib/perl/ /usr/local/bin/youri-submit --verbose --config /etc/youri/submit-queue.conf --define user=$user --define prefix=$prefix --define section=$section $target @packages &> $done/$path/$prefix.youri";
+
+    plog("running $command");
+    if (!system($command)) {
+	plog("upload succeeded.");
+
+	# now check if we need to keep the current srpm in todo
+	my $all_uploaded = 1;
+
+	foreach my $arch (@{$config->{arch}}) {
+	    if (!$cache->{arch}{$prefix}{$arch}) {
+		$all_uploaded = 0;
+	    }
+	}
+
+	if ($all_uploaded) {
+	    plog("cleaning upload tree for $prefix");
+	    # remove srpm
+	    # remove lock
+	}
+    } else {
+	# should send a mail or something
+	plog("upload failed ($!), rejecting files in $reject/$path/");
 	foreach my $rpm (@{$pkg_tree{$prefix}{rpms}}) {
-	    push @packages, "$done/$path/${prefix}_$rpm"
+	    my $err = link "$done/$path/${prefix}_$rpm", "$reject/$path/${prefix}_$rpm";
+	    plog("ERROR: link of $rpm failed ($!)");
 	}
-	$user ||= $config->{upload_user};
-	my $command = "/usr/bin/sudo -H -u $config->{upload_user} /usr/bin/perl -I/usr/local/lib/perl/ /usr/local/bin/youri-submit --verbose --config /etc/youri/submit-queue.conf --define user=$user --define prefix=$prefix --define section=$section $target @packages &> $done/$path/$prefix.youri";
-	print {$run{LOG}} "$program_name: running $command\n";
-	if (!system($command)) {
-	    print {$run{LOG}} "$program_name: upload succeeded.\n";
-	    # now check if we need to keep the current srpm in todo
-	    my $all_uploaded = 1;
-	    foreach my $arch (@{$config->{arch}}) {
-		if (!$cache->{arch}{$prefix}{$arch}) {
-		    $all_uploaded = 0
-		}
-	    }
-	    if ($all_uploaded) {
-		print {$run{LOG}} "$program_name: cleaning upload tree for $prefix\n";
-		# remove srpm
-		# remove lock
-	    }
-	} else {
-	    # should send a mail or something
-	    print {$run{LOG}} "$program_name: upload failed ($!), rejecting files in $reject/$path/\n";
+	link "$done/$path/$prefix.youri", "$reject/$path/$prefix.youri";
+
+	my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+/;
+	if ($user) {
+	    my $text = qq{The upload of the following packages failed:\n};
+	    my $rpms;
 	    foreach my $rpm (@{$pkg_tree{$prefix}{rpms}}) {
-		my $err = link "$done/$path/${prefix}_$rpm", "$reject/$path/${prefix}_$rpm";
-		print {$run{LOG}} "ERROR $program_name: link of $rpm failed ($!)\n"
+		$rpm =~ /src\.rpm$/ or next;
+		$rpms .= "$rpm ";
+		$text .= "- $rpm\n"
 	    }
-	    link "$done/$path/$prefix.youri", "$reject/$path/$prefix.youri";
-	    my ($user) = $prefix =~ /\d{14}\.(\w+)\.\w+\.\d+/;
-	    if ($user) {
-		my $text = qq{The upload of the following packages failed:\n};
-		my $rpms;
-		foreach my $rpm (@{$pkg_tree{$prefix}{rpms}}) {
-		    $rpm =~ /src\.rpm$/ or next;
-		    $rpms .= "$rpm ";
-		    $text .= "- $rpm\n"
-		}
-		my $to =  $mail->{$user} || "$user\@mandriva.com, $user\@mandriva.org";
-		$text .= "\nUpload log available in $config->{http_queue}/rejected/$path/$prefix.youri\n";
-		sendmail($to, $config->{admin} , "Upload failed for $rpms", $text, "Emi the upload bot <$config->{admin}>", 0)
-	    }
-	    # should delete the files
+	    my $to =  $mail->{$user} || "$user\@mandriva.com, $user\@mandriva.org";
+	    $text .= "\nUpload log available in $config->{http_queue}/rejected/$path/$prefix.youri\n";
+	    sendmail($to, $config->{admin} , "Upload failed for $rpms", $text, "Emi the upload bot <$config->{admin}>", 0)
 	}
-	# delete the files which should have heen either put in queue or rejected
-	unlink $_ foreach @packages;
-	# keep the log file for debugging
-	# unlink "$done/$path/$prefix.youri";
-	# unlink the sources rpm, other arch will be able to grab it into the repository
-	print "unlink $todo/$path/${prefix}_$_\n" foreach @{$pkg_tree{$prefix}{todo}};
-	unlink "$todo/$path/${prefix}_$_" foreach @{$pkg_tree{$prefix}{todo}}
-    } else {
-	print {$run{LOG}} "WARNING $program_name: mandatory arch (", join(' ', keys %missing),") are missing for $prefix, waiting\n"
+
+	# should delete the files
+    }
+
+    # delete the files which should have heen either put in queue or rejected
+    unlink $_ foreach @packages;
+
+    # keep the log file for debugging
+    unlink "$done/$path/$prefix.youri";
+
+    # unlink the sources rpm, other arch will be able to grab it into
+    # the repository
+    foreach (@{$pkg_tree{$prefix}{todo}}) {
+	plog("unlink $todo/$path/${prefix}_$_");
+	unlink "$todo/$path/${prefix}_$_";
     }
 }
 
-- 
cgit v1.2.1