diff options
| -rwxr-xr-x | iurt2 | 294 |
1 files changed, 163 insertions, 131 deletions
@@ -303,7 +303,9 @@ if (!$run{debug} && $run{media} || $run{chroot}) { my $debug_tag = '_debug' if $run{debug}; my $chroot_name = "chroot_$run{distro_tag}$debug_tag"; my $chroot = "$config->{local_home}/$chroot_name"; +$run{chroot_path} = $chroot; my $chroot_tar = "$chroot.$run{my_arch}.tar.gz"; +$run{chroot_tar} = $chroot_tar; if ($run{chroot}) { check_chroot($chroot, $chroot_tar, \%run) or die "FATAL iurt: could not prepare initial chroot" } @@ -412,7 +414,7 @@ if ($run{shell}) { ($union_id, my $unionfs_tmp, my $chroot_tmp) = create_temp_chroot(\%run, $cache, $unionfs_tmp, $unionfs_dir, $union_id) or die "FATAL iurt: could not create temporary chroot"; add_local_user($chroot_tmp, \%run, $luser, $run{uid}) or die "FATAL iurt: could not add local user"; $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp"; - add_packages(\%run, $config, $chroot_tmp, $luser, 'sudo', 'urpmi'); + add_packages(\%run, $config, $chroot_tmp, $luser, 'sudo', 'urpmi') or die "FATAL iurt: could not add urpmi and sudo in the chroot"; add_sudoers(\%run, $luser); if ($run{shell}) { print {$run{LOG}} "iurt: dumping to a chrooted shell into $chroot_tmp\n"; @@ -484,6 +486,11 @@ do { print {$run{LOG}} "iurt: packages $srpm [$done/$to_compile]\n"; # FIXME unfortunately urpmi stalls quite often my $retry = 0; + # + # current rpm is sometime segfaulting, and iurt is them blocked and cannot + # + # $cache->{failure}{$srpm} = 1; + # dump_cache(\%run); retry: if (!$run{chrooted_urpmi}) { my $match = "urpmi $urpmi_options --root $chroot_tmp"; @@ -496,8 +503,8 @@ do { if ($run{chrooted_urpmi}) { $run{urpmi_command} = "urpmi $urpmi_options --root $chroot_tmp "; - add_packages(\%run, $config, $chroot_tmp, $luser, 'urpmi'); - add_media(\%run, $config, $chroot_tmp); + add_packages(\%run, $config, $chroot_tmp, $luser, 'urpmi') or next; + add_media(\%run, $config, $chroot_tmp, 'Main', "--distrib $run{chrooted_media}") or next; if (-d "$local_media/hdlist.cz") { mkdir("$chroot_tmp/iurt_media/"); opendir my $dir, $local_media; @@ -513,7 +520,7 @@ do { } } next if $next; - add_local_media(\%run, $config, $chroot_tmp, "/iurt_media") + add_media(\%run, $config, $chroot_tmp, 'iurt_group', "iurt_group file:///iurt_media") or next } $run{urpmi_command} = "chroot $chroot_tmp urpmi $urpmi_options " } else { @@ -555,58 +562,59 @@ do { # # on x86_64 the rpm database is getting corrupted and sometimes rpm do not found anymore # installed packages, retrying several time to be sure something is really broken - my $install_deps_retry = 5; - while (!perform_command("sudo $run{urpmi_command} $path_srpm/$srpm", + if (!perform_command("sudo $run{urpmi_command} $path_srpm/$srpm", \%run, $config, mail => $config->{admin}, error => "[REBUILD] install of build dependencies of $srpm failed on $run{my_arch}", hash => "install_deps_$srpm", - timeout => 900, + timeout => 600, srpm => $srpm, freq => 1, #cc => $cc, - retry => 10, + retry => 2, debug_mail => $run{debug}, error_regexp => 'cannot be installed', - wait_regexp => 'database locked', - wait_callback => $wait_urpmi, - log => "$local_spool/log/$srpm/", - reopen_log => 1, - callback => sub { - my ($opt, $output) = @_; - print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug}; + wait_regexp => { + 'database locked' => $wait_urpmi, + 'is needed by' => sub { + print {$run{LOG}} "WARNING iurt: rpm database seems corrupted, retrying\n"; + system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null"); + foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") } }, + }, + wait_callback => $wait_urpmi, + log => "$local_spool/log/$srpm/", + callback => sub { + my ($opt, $output) = @_; + print {$run{LOG}} "calling callback for $opt->{hash}\n" if $run{debug}; # 20060614 it seems the is needed urpmi error is due to something else (likely a database corruption error). # my @missing_deps = $output =~ /(?:(\S+) is needed by )|(?:\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\))/g; - my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g; - # as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway - @missing_deps or return 1; - while (my $missing_deps = shift @missing_deps) { - my $version = shift @missing_deps; - my $version2 = shift @missing_deps; - $version ||= $version2 || 0; - my $p = $pack_provide{$missing_deps} || $missing_deps; - my $other_maint = `rpmmon -p "$p"`; - chomp $other_maint; - print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n"; - if ($other_maint && $other_maint ne 'NOT_FOUND') { - $opt->{mail} = "$maintainer, $other_maint"; - $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}"; - } - # remember what is needed, and do not try to recompile until it is available - push @{$cache->{needed}{$srpm}}, [ $missing_deps, $version, $other_maint || $maintainer ]; - } - 0 - }, - )) { - $install_deps_retry--; - foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") } - if (!$install_deps_retry || $cache->{needed}{$srpm}) { + my @missing_deps = $output =~ /\(due to unsatisfied ([^[ ]*)(?: (.*)|\[(.*)\])?\)/g; + # as it seems that rpm db corruption is making urpmi returning false problem on deps installation, try to compile anyway + @missing_deps or return 1; + while (my $missing_deps = shift @missing_deps) { + my $version = shift @missing_deps; + my $version2 = shift @missing_deps; + $version ||= $version2 || 0; + my $p = $pack_provide{$missing_deps} || $missing_deps; + my $other_maint = `rpmmon -p "$p"`; + chomp $other_maint; + print {$run{LOG}} "missing dep: $missing_deps ($other_maint)\n"; + if ($other_maint && $other_maint ne 'NOT_FOUND') { + $opt->{mail} = "$maintainer, $other_maint"; + $opt->{error} = "[MISSING] $missing_deps, needed to build $srpm, is not available on $run{my_arch}"; + } + # remember what is needed, and do not try to recompile until it is available + $cache->{needed}{$srpm}{$missing_deps} = { version => $version, maint => $other_maint || $maintainer }; + } + 0 + }, + )) { $run{status}{$srpm} = 'install_deps_failure'; - next + next; } - } # try to workarround the rpm -qa db4 error(2) from dbcursor->c_get: No such file or directory # running rpm -qa several time seems to fix the problem + system("sudo chroot $chroot_tmp rm -rf /var/lib/rpm/__db* &> /dev/null"); foreach (1 .. 3) { system("sudo chroot $chroot_tmp rpm -qa &> /dev/null") } perform_command("sudo chroot $chroot_tmp rpm -qa", \%run, $config, @@ -623,7 +631,7 @@ do { } if (!perform_command(qq{TMP=/home/$luser/tmp/ sudo chroot $chroot_tmp /bin/su $luser -c "$command"}, \%run, $config, - mail => $maintainer, + # mail => $maintainer, error => "[REBUILD] $srpm from $run{distro_tag} does not build correctly on $run{my_arch}", hash => "build_$srpm", timeout => 18000, @@ -670,7 +678,10 @@ do { goto retry } $cache->{failure}{$srpm} = 1; + # 20060615 $run{status}{$srpm} = 'build_failure'; + dump_cache(\%run); + dump_status($local_spool, \%run); next } } @@ -678,21 +689,22 @@ do { delete $cache->{needed}{$srpm} if defined $cache->{needed}{$srpm}; if (!perform_command("sudo urpmi $urpmi_options --root $chroot_tmp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm", \%run, $config, - mail => $maintainer, + # mail => $maintainer, error => "[REBUILD] binaries packages generated from $srpm do not install correctly", hash => "binary_test_$srpm", srpm => $srpm, timeout => 600, debug_mail => $run{debug}, freq => 1, - wait_regexp => 'database locked', - wait_callback => $wait_urpmi, + wait_regexp => { 'database locked' => $wait_urpmi }, error_regexp => 'unable to access', log => "$local_spool/log/$srpm/")) { $cache->{failure}{$srpm} = 1; $run{status}{$srpm} = 'binary_test_failure'; next } + $run{status}{$srpm} = 'ok'; + delete $cache->{failure}{$srpm}; if ($run{debug}) { print {$run{LOG}} "iurt: debug mode, skip other packages\n"; exit @@ -703,7 +715,6 @@ do { system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_media &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_media ($!)\n"; } else { print {$run{LOG}} "iurt: build successful, copying packages to $local_spool.\n"; - $run{status}{$srpm} = 'ok'; system("cp $chroot_tmp/home/$luser/rpm/RPMS/*/*.rpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy rpm files from $chroot_tmp/home/$luser/rpm/RPMS/ to $local_spool ($!)\n"; if ($run{copy_srpm}) { system("cp $chroot_tmp/home/$luser/rpm/SRPMS/$srpm $local_spool &>/dev/null") and print {$run{LOG}} "ERROR: could not copy $srpm from $chroot_tmp/home/$luser/rpm/SRPMS/ to $local_spool ($!)\n"; @@ -731,18 +742,7 @@ if (open my $file, ">$local_spool/log/wrong_srpm_names.log") { close $file } -if (open my $file, ">$local_spool/log/status.$run{media}.log") { - foreach my $srpm (keys %{$run{status}}) { - print $file "$srpm: "; - if ($run{status}{$srpm}) { - print $file $run{status}{$srpm} - } else { - print $file "unknown" - } - print $file "\n" - } - close $file -} +dump_status($local_spool, \%run); if ($config->{rsync_to} && !$run{no_rsync}) { # remove some old and very big log files not to saturate the server @@ -846,11 +846,13 @@ sub clean { my ($var, $cmd, $kill_cmd, $verbose) = @_; my $ps; my $i; + use POSIX ":sys_wait_h"; while ($ps = `$cmd "$var"`) { system(qq{$kill_cmd "$var" &>/dev/null}); sleep 1; $ps =~ s/\n/,/g; print {$run{LOG}} "Trying to remove previous blocked processes for $var ($ps)\n" if $verbose > 1; + waitpid(-1, WNOHANG); return 0 if $i++ > 10 } 1 @@ -859,7 +861,7 @@ sub clean { sub clean_unionfs { my ($unionfs_dir, $run, $r, $union_id) = @_; -d "$unionfs_dir/unionfs.$r.$union_id" or return $union_id; - print {$run->{LOG}} "Cleaning $unionfs_dir/unionfs.$run.$union_id\n" if $run->{verbose} > 1; + print {$run->{LOG}} "Cleaning $unionfs_dir/unionfs.$r.$union_id\n" if $run->{verbose} > 1; my $nok = 1; my $proc = "$unionfs_dir/unionfs.$r.$union_id/proc"; while ($nok) { @@ -926,25 +928,25 @@ sub check_mounted { sub check_needed { my ($srpm) = @_; if (!defined $cache->{needed}{$srpm} && !ref $cache->{needed}{$srpm}) { return 1 } - my @n; my $ok = 1; - foreach my $t (@{$cache->{needed}{$srpm}}) { - my ($name, $version, $maint) = @$t; + foreach my $name (keys %{$cache->{needed}{$srpm}}) { + my ($version, $maint) = $cache->{needed}{$srpm}{'version','maint'}; my ($p_version) = $provides{$name}; if ($p_version) { next if $version == $p_version; next if URPM::ranges_overlap($version, $p_version) } $ok = 0; - push @n, [ $name, $version ]; + if ($version) { + $cache->{needed}{$srpm}{$name}{version} = $version; + } my $v ||= $version; print {$run{LOG}} "ERROR iurt: $srpm needs $name $v to be compiled.\n"; # try to recompile it once in a while - last if ! $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72; + last if $cache->{warning}{"install_deps_$srpm"}{$maint}++ % 72; return 1 } delete $cache->{needed}{$srpm}; - $cache->{needed}{$srpm} = \@n if @n; $ok } @@ -1090,18 +1092,22 @@ sub perform_command { $opt{freq} ||= 24; print {$run{LOG}} "Timeout $opt{timeout}\n" if $run{verbose} > 2; # from alarm perldoc - my ($output, $comment); + my ($output, $fulloutput, $comment); my ($kill, $pipe); if ($opt{debug}) { print "Would have rum $command with a timeout of $opt{timeout}\n"; return 1 } local $SIG{PIPE} = sub { print "Broken pipe!\n"; $pipe = 1 }; - my $logfile = "$opt{log}/$opt{hash}.$run->{run}.log"; my $retry = $opt{retry} || 1; my $call_ret = 1; - my ($err, $pid); + my ($err, $pid, $try); + my $logfile = "$opt{log}/$opt{hash}.$run->{run}.log"; while ($retry) { + $try++; + if ($opt{retry} > 1) { + $logfile = "$opt{log}/$opt{hash}-$try.$run->{run}.log"; + } if ($opt{log}) { my $parent_pid = $$; $pid = fork; @@ -1140,11 +1146,7 @@ sub perform_command { print {$run{LOG}} "$command\n" if $run{verbose} > 2; if ($opt{log}) { #$output = `$command 2>&1 2>&1 | tee $opt{log}/$opt{hash}.$run.log`; - if ($opt{reopen_log}) { - system("$command 2>&1 >> $logfile"); - } else { - system("$command >& $logfile"); - } + system("$command 2>&1 >> $logfile"); } else { $output = `$command 2>&1`; } @@ -1153,9 +1155,7 @@ sub perform_command { $err = $?; # kill pid watching log file size if ($pid) { - use POSIX ":sys_wait_h"; - kill 14, $pid; - waitpid(-1, WNOHANG) + kill_for_good($pid); } if ($@) { # timed out @@ -1166,6 +1166,7 @@ sub perform_command { local $/; $output = <$log> } + $fulloutput .= $output; if (ref $opt{callback}) { $call_ret = $opt{callback}(\%opt, $output); $call_ret == -1 and return 1; @@ -1182,27 +1183,31 @@ sub perform_command { $comment = "Command failed: $command\n" } # Maybe this has to be put before all the commands altering the $output var - if ($opt{wait_regexp} && $output =~ /$opt{wait_regexp}/m) { - $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback}; - print {$run->{LOG}} "ERROR iurt: $opt{wait_regexp} !\n"; - sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$opt{wait_regexp}\n\n$comment\n$output", 0, 0, $opt{debug_mail}); - if ($opt{die}) { - dump_cache($run); - die "FATAL iurt: $opt{error}." + my $inc; + if ($opt{wait_regexp}) { + foreach my $wr (keys %{$opt{wait_regexp}}) { + if ($output =~ /$wr/m) { + $inc = $opt{wait_callback}(\%opt, $output) if ref $opt{wait_callback}; + print {$run->{LOG}} "ERROR iurt: $wr !\n"; + sendmail($config->{admin}, '' , "$opt{hash} on $run->{my_arch} for $run->{media}: could not proceed", "$wr\n\n$comment\n$output", 0, 0, $opt{debug_mail}); + } } - $retry--; - return if !$retry - } else { + } + if ($inc) { + $retry += $inc + } elsif ($call_ret && !$kill && !$err && !$opt{error_regexp} || $fulloutput !~ /$opt{error_regexp}/) { $retry = 0 + } else { + $retry-- } } - if (!$call_ret || $kill || $err || $opt{error_regexp} && $output =~ /$opt{error_regexp}/) { + if (!$call_ret || $kill || $err || $opt{error_regexp} && $fulloutput =~ /$opt{error_regexp}/) { if ($opt{log} && $config->{log_url}) { $comment = qq|See $config->{log_url}/$run{distro_tag}/$run{my_arch}/log/$opt{srpm}/\n\n$comment| } my $out; - if (length $output < 10000) { - $out = $output + if (length $fulloutput < 10000) { + $out = $fulloutput } else { $out = "Message too big, see http link for details\n" } @@ -1216,7 +1221,7 @@ sub perform_command { } } $cache->{warning}{$opt{hash}}{$opt{mail}}++; - print {$run->{LOG}} "\n$comment\n$output\n"; + print {$run->{LOG}} "\n$comment\n$fulloutput\n"; if ($opt{die}) { dump_cache($run); die "FATAL iurt: $opt{error}." @@ -1228,19 +1233,25 @@ sub perform_command { sub kill_for_good { my ($pid) = @_; + use POSIX ":sys_wait_h"; kill 14, $pid; - sleep 2; + sleep 1; + waitpid(-1, WNOHANG); if (getpgrp $pid != -1) { kill 15, $pid; - sleep 2; + sleep 1; + waitpid(-1, WNOHANG); if (getpgrp $pid != -1) { - kill 9, $pid + print STDERR "WARNING iurt: have to kill -9 pid $pid\n"; + kill 9, $pid; + sleep 1; + waitpid(-1, WNOHANG); } } } sub check_chroot { - my ($chroot, $chroot_tar, $run) = @_; + my ($chroot, $chroot_tar, $run, $opt) = @_; print {$run{LOG}} "iurt: checking basesystem tar\n" if $run{verbose}; system(qq{sudo pkill -9 -u root -f "urpmi $urpmi_options --root $chroot" &> /dev/null}); if (!clean_chroot($chroot, $run, 1)) { @@ -1294,10 +1305,6 @@ sub check_pid { open my $lock2, ">$lockfile.2" or die "FATAL iurt: could not open lock file $lockfile.2"; print $lock2 $$; close $lock2; - } else { - # protection mecchanism to remove dead lock files - unlink "$lockfile.2"; - die "FATAL iurt: could not lock pid file (status $status $!)\n"; } } if (!$run->{concurrent_run}) { @@ -1480,7 +1487,7 @@ sub add_local_user { # if (system(qq|sudo chroot $chroot_tmp usermod -u $run{uid} builder|)) { if (system(qq|sudo chroot $chroot_tmp useradd -M -u $uid $luser|) || system(qq|sudo chroot $chroot_tmp id $luser|)) { print {$run->{LOG}} "ERROR: setting userid $uid to $luser in $chroot_tmp failed, trying to check the chroot\n"; - check_chroot($chroot, $chroot_tar, $run); + check_chroot($run->{chroot_path}, $run->{chroot_tar}, $run); return } if (system(qq|sudo chroot $chroot_tmp cp -R /home/builder /home/$luser|)) { @@ -1551,22 +1558,35 @@ sub create_temp_chroot { sub add_packages { my ($run, $config, $chroot, $user, @packages) = @_; my $f; - perform_command("sudo $run{urpmi_command} @packages", + if (!perform_command("sudo $run{urpmi_command} @packages", $run, $config, mail => $config->{admin}, - timeout => 600, + timeout => 120, freq => 1, - retry => 10, + retry => 2, debug_mail => $run->{debug}, error_regexp => 'cannot be installed', - wait_regexp => 'database locked', - wait_callback => sub { - print {$run->{LOG}} "WARNING iurt: urpmi database locked, waiting...\n"; - sleep 30; - $wait_limit++; - if ($wait_limit > 10) { - $wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}) - } },) + wait_regexp => { + 'is needed by' => sub { + print {$run{LOG}} "WARNING iurt: rpm database seems corrupted, retrying\n"; + system("sudo chroot $chroot rm -rf /var/lib/rpm/__db* &> /dev/null"); + foreach (1 .. 3) { system("sudo chroot $chroot rpm -qa &> /dev/null") } + 1 + }, + 'database locked' => sub { + print {$run->{LOG}} "WARNING iurt: urpmi database locked, waiting...\n"; + sleep 30; + $wait_limit++; + if ($wait_limit > 10) { + $wait_limit = 0; system(qq{sudo pkill -9 urpmi &>/dev/null}); + return + } + 1 + } },)) { + print {$run->{LOG}} "ERROR iurt: could not install @packages inside $chroot\n"; + return 0 + } + 1 } sub add_sudoers { @@ -1604,37 +1624,26 @@ sub remove_chroot { } sub add_media { - my ($run, $config, $chroot) = @_; + my ($run, $config, $chroot, $regexp, $media) = @_; print {$run->{LOG}} "iurt: adding media $run{chrooted_media} in chroot $chroot\n"; - if (!perform_command("sudo chroot $chroot urpmi.addmedia --distrib $run{chrooted_media}", - $run, $config, - mail => $config->{admin}, - timeout => 120, - freq => 1, - retry => 5, - debug_mail => $run->{debug})) { - } - check_media_added($chroot, 'Main') or die "FATAL iurt: could not add media into the chroot" -} - -sub add_local_media { - my ($run, $config, $chroot, $local_media) = @_; - print {$run->{LOG}} "iurt: adding media $local_media in chroot $chroot\n"; - if (perform_command("sudo chroot $chroot urpmi.addmedia iurt_group file://$local_media", + if (!perform_command("sudo chroot $chroot urpmi.addmedia $media", $run, $config, mail => $config->{admin}, timeout => 120, freq => 1, - retry => 5, + retry => 2, debug_mail => $run->{debug})) { } - check_media_added($chroot, 'iurt_group') or die "FATAL iurt: could not add local media into the chroot" + if (!check_media_added($chroot, $regexp)) { + print "ERROR iurt could not add media into the chroot\n"; + return + } + 1 } sub check_media_added { my ($chroot, $media) = @_; my $medias = `sudo chroot $chroot urpmi.removemedia 2>&1`; - print "MEDIA $medias\n"; $medias =~ /one of.* $media/m } @@ -1669,10 +1678,16 @@ sub recreate_srpm { debug_mail => $run->{debug}, hash => "install_$srpm", retry => $retry, + wait_regexp => { + 'cannot write to %sourcedir /usr/src/rpm/SOURCES' => sub { + print {$run->{LOG}} "ERROR iurt: chroot seems corrupted, try to rebuild it\n"; + check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run) + } + }, callback => sub { my ($opt, $output) = @_; print {$run->{LOG}} "calling callback for $opt->{hash}\n" if $run->{debug}; - if ($output =~ /warning: group .* does not exist - using root/i) { + if ($output =~ /warning: (group|user) .* does not exist - using root/i) { return 1 } elsif ($output =~ /user $luser does not exist/) { print {$run->{LOG}} "WARNING iurt: chroot seems corrupted...\n" if $run->{verbose} > 1; @@ -1686,7 +1701,7 @@ sub recreate_srpm { \%run, $config, %opt)) { print {$run->{LOG}} "ERROR iurt: chrooting failed (retry $opt{retry}\n" if $run->{debug}; if ($opt{retry}) { - check_chroot($chroot, $chroot_tar, \%run) or return; + check_chroot($run->{chroot_path}, $run->{chroot_tar}, \%run) or return; return -1 } return @@ -1767,3 +1782,20 @@ sub order_packages { $run->{todo} = [ sort { $a->[2] <=> $b->[2] } @packages ]; @packages } + +sub dump_status { + my ($local_spool, $run) = @_; + if (open my $file, ">$local_spool/log/status.$run->{media}.log") { + foreach my $srpm (sort keys %{$run->{status}}) { + print $file "$srpm: "; + if ($run{status}->{$srpm}) { + print $file $run->{status}{$srpm} + } else { + print $file "unknown" + } + print $file "\n" + } + close $file + } +} + |