path: root/gendistrib

#!/usr/bin/perl

(our $VERSION) = q$Id$ =~ /(\d+\.\d+)/;

#- Copyright (C) 1999-2005 Mandrakesoft
#-
#- This program is free software; you can redistribute it and/or modify
#- it under the terms of the GNU General Public License as published by
#- the Free Software Foundation; either version 2, or (at your option)
#- any later version.
#-
#- This program is distributed in the hope that it will be useful,
#- but WITHOUT ANY WARRANTY; without even the implied warranty of
#- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#- GNU General Public License for more details.
#-
#- You should have received a copy of the GNU General Public License
#- along with this program; if not, write to the Free Software
#- Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

use strict;
use Cwd;
use URPM;
use URPM::Build;
use Getopt::Long;
use Distribconf::Build;
use Packdrakeng;
use Pod::Usage;

my $urpm = new URPM;
my $tempdir = -d $ENV{TMPDIR} ? $ENV{TMPDIR} : -d "$ENV{HOME}/tmp" ? "$ENV{HOME}/tmp" : "/tmp";
my $headers_dir = $tempdir . "/.build_hdlist";

sub usage () {
    pod2usage({ -verbose => 1 });
}

my %urpmfiles;

GetOptions(
    'blind' => \my $blind,
    'compss=s' => \$urpmfiles{compss},
    'depslist=s' => \$urpmfiles{depslist},
    'destdir=s' => \my $destdir,
    'hdlists=s' => \$urpmfiles{hdlists},
    'headersdir=s' => \$headers_dir,
    'help|h' => sub { usage(); exit 0; },
    'mediacfg=s' => \$urpmfiles{mediacfg},
    'nobadrpm' => \my $dontdie,
    'nochkdep' => \my $nochkdep,
    'noclean' => \my $noclean,
    'noemptymedia' => \my $noemptymedia,
    'nomd5sum' => \my $nomd5sum,
    'nomediainfo' => \my $nomediainfo,
    'provides=s' => \$urpmfiles{provides},
    'skipmissingdir' => \my $skipmissingdir,
    's' => \my $nooutput,
    'v|version' => sub { warn "$0 version $VERSION\n"; exit 0 },
);

my @root = grep { $_ } @ARGV;

@root > 0 or do { usage(); exit 1 };

my $distrib = Distribconf::Build->new($root[0]);

$distrib->loadtree or die "$root[0] does not seem to be a distribution tree\n";

if (defined($urpmfiles{mediacfg})) {
    $distrib->parse_mediacfg($urpmfiles{mediacfg}) or die "Can't read $urpmfiles{mediacfg}\n";
} elsif (defined($urpmfiles{hdlists})) {
    $distrib->parse_hdlists($urpmfiles{hdlists}) or die "Can't read $urpmfiles{hdlists}\n";
} else {
    $distrib->parse_mediacfg || $distrib->parse_hdlists or die "Can't read the dsitrib config\n";
}

my $destinfodir = $destdir ? 
    $destdir . '/' . $distrib->getpath(undef, "infodir") :
    $distrib->getfullpath(undef, "infodir");

my %default_urpmfiles = (
    depslist => $destinfodir . "/depslist.ordered",
    provides => $destinfodir . "/provides",
    compss =>   $destinfodir . "/compss",
    version =>  $destdir ?
        $destdir . $distrib->getpath(undef, "VERSION") : 
        $distrib->getfullpath(undef, "VERSION"),
    md5sum =>   $destinfodir . "/MD5SUM",
);

while (my ($k, $v) = each(%default_urpmfiles)) {
    $urpmfiles{$k} ||= $v;
}

$distrib->check(\*STDERR) unless $nooutput;

my @hdlists;
my @media_mising_dirs;
foreach ($distrib->listmedia) {
    $distrib->getvalue($_, 'askmedia') || $distrib->getvalue($_, 'suppl') and next;

    
    if (! -d ($distrib->getfullpath($_, 'path'))) {
        if ($skipmissingdir) {
            printf(STDERR
                "Skipping missing media %s\n",
                $distrib->getpath($_, 'path')
            ) unless $nooutput;
        } else {
            # delaying error report to report all errors, not the only first
            push(@media_mising_dirs, $_);
        }
        next;
    }

    push @hdlists, {
        synthesis => $destdir ?
            $destdir . '/' . $distrib->getpath($_, 'synthesis') :
            $distrib->getfullpath($_, 'synthesis'),
        hdlist => $destdir ?
            $destdir . '/' .$distrib->getpath($_, 'hdlist') :
            $distrib->getfullpath($_, 'hdlist'),
        dir => $distrib->getpath($_, 'path'),
        descr => $distrib->getvalue($_, 'name'),
        mediainfo => $destdir ?
            $destdir . '/' . $distrib->getpath(undef, 'infodir') :
            $distrib->getfullpath(undef, 'infodir'),
        thismediainfo => ($destdir ?
            $destdir . '/' . $distrib->getpath($_, 'path') :
            $distrib->getfullpath($_, 'path')) . "/media_info",
        synthesis2 => ($destdir ?
            $destdir . '/' . $distrib->getpath($_, 'path') :
            $distrib->getfullpath($_, 'path')) . "/media_info/synthesis.hdlist.cz",
        hdlist2 => ($destdir ?
            $destdir . '/' . $distrib->getpath($_, 'path') :
            $distrib->getfullpath($_, 'path')) . "/media_info/hdlist.cz",
        md5sum => ($destdir ?
            $destdir . '/' . $distrib->getpath($_, 'path') :
            $distrib->getfullpath($_, 'path')) . "/media_info/MD5SUM",
    };
}

# if there is result here, $skipmissingdir is not set and there is errors:
if (@media_mising_dirs) {
    foreach my $media (@media_mising_dirs) {
        printf(STDERR  
            "Missing dir '%s' for media '%s'\n", 
            $distrib->getpath($media, 'path'), 
            $distrib->getvalue($media, 'name')
        ) unless $nooutput;
    }
    die "Stopping because dirs are missing, specify --skipmissingdir to ignore\n";
}

# Creating destination directory, doing it early,
# don't die after 30 minutes of rpm parsing
if (!-d $destinfodir) {
    mkdir $destinfodir, 0755
        or die qq(Can't create directory "$destinfodir": $!\n);
}

foreach my $e (@hdlists) {
	for my $d (qw(mediainfo thismediainfo)) {
	    if (! -d $e->{$d}) {
		mkdir $e->{$d}, 0755
		    or die qq(Can't create directory "$e->{$d}": $!\n);
	    }
	}
}

sub clean_cache {
    unless ($noclean) {
	system($ENV{LD_LOADER} ? $ENV{LD_LOADER} : @{[]}, "rm", "-rf", $headers_dir);
	mkdir $headers_dir
	    or die qq(Can't create directory "$headers_dir": $!\n);
    }
}

clean_cache();

foreach (0..$#hdlists) {
    my $e = $hdlists[$_];
    my $r;

    #- try to find the right repository where can be found the directory
    #- listed in the hdlist file.
    #- if the number of roots is equal to the number of media, assume one
    #- media per root, else try to find a valid root containing the media.
    $r ||= $root[0];
    if (scalar(@hdlists) == scalar(@root)) {
	$r = $root[$_];
    } else {
	foreach (@root) {
	    -d "$_/$e->{dir}" and $r = $_, last;
	}
    }

    #- fake build of architecture dependent directory.
    my @files;
    if ($e->{dir} =~ /%{ARCH}/) {
	foreach my $arch (qw(i686 i586 i486 i386 k8 k7 k6 amd64 amd32 x86_64 x86_32 ia64 ia32
                             ppc sparc sparc32 sparc64 alpha noarch)) {
	    my $dir = $e->{dir};
	    $dir =~ s|%{ARCH}|$arch|g;
	    push @files, glob("$r/$dir/*.$arch.rpm");
	}
    } else {
	push @files, glob("$r/$e->{dir}/*.rpm");
    }
    @files or do { 
        print STDERR "unable to find rpm files in $e->{dir}\n" unless $nooutput;
        next;
    };

    print STDERR "parsing rpm files in directory $r/$e->{dir}\n" unless $nooutput;
    my @headers = $urpm->parse_rpms_build_headers(
	dir  => $headers_dir,
	rpms => \@files,
	dontdie => $dontdie,
	silent => $nooutput,
    );
    # TODO if @headers is empty ?
    $e->{headers} = \@headers;
    
    if (!$blind) {
        # checking if hdlist rebuild is need
        print STDERR "Checking if hdlist need to be rebuild for media $e->{descr}\n" unless $nooutput;
        if(!compare_headers_with_hdlist($e->{hdlist}, @headers)) {
            $e->{noneedrebuild} = 1;
            print "No\n" unless $nooutput;
        } else {
            print "Yes\n" unless $nooutput;
        }
    }
}

# return 1 if differ, 0 otherwise
sub compare_headers_with_hdlist {
    my ($hdlist, @headers) = @_;
    if (my $pack = Packdrakeng->open(archive => $hdlist)) {
        my %exists_headers;
        $exists_headers{$_} = 0 foreach(@headers);
        my (undef, $files, undef) = $pack->getcontent();
        foreach my $file (@{$files || []}) {
            if (exists($exists_headers{$file})) {
                # TODO checking somethings else that name
                delete($exists_headers{$file});
            } else {
                # one file is in hdlist, not in our headers => differ
                return 1;
            }
        }
        # if the same files are in hdlist, and headers list,
        # nothing left
        if (keys %exists_headers) {
            return 1;
        }
    } else {
        # no valid hdlist, it differ for sure !
        return 1;
    }
    return 0; # no diff
}

if ($noemptymedia) {
    foreach my $e (@hdlists) {
        $e->{headers} or die "Empty media were found, stopping\n";
    }
}

#- clean everything to start second pass.
print STDERR "clean data for second pass\n" unless $nooutput;
$urpm->unresolved_provides_clean;

#- temporary file where to build hdlists
my $temp_hdlist = $tempdir . '/hdlist' . $$;
foreach (0..$#hdlists) {
    my $e = $hdlists[$_];

    if ($e->{headers}) { # We have rpms in this media
    
    print STDERR qq(parsing headers for "$e->{descr}"\n) unless $nooutput;
    my ($start, $end) = $urpm->parse_headers(dir     => $headers_dir,
					     headers => $e->{headers},
				             dontdie => $dontdie,
					     silent  => $nooutput);

    print STDERR "computing deps\n" unless $nooutput;
    $urpm->compute_deps;

    if ($e->{noneedrebuild}) {
        # No media change, nothing to write
        next;
    }
    
    print STDERR qq(building hdlist for medium "$e->{descr}"\n) unless $nooutput;
    unlink $temp_hdlist;
    $urpm->build_hdlist(start  => $start,
			end    => $end,
			dir    => $headers_dir,
			hdlist => $temp_hdlist,
			ratio  => 9);
    system('/bin/mv', $temp_hdlist, $e->{hdlist});

    print STDERR qq(building synthesis for medium "$e->{descr}"\n) unless $nooutput;
    $urpm->build_synthesis(start     => $start,
			   end       => $end,
			   synthesis => $e->{synthesis});

    } else { # no rpm, creating empty but valid index
        my $pack = Packdrakeng->new(archive =>  $temp_hdlist);
        $pack = undef; # closing archive

        system('/bin/mv', $temp_hdlist, $e->{hdlist});
        open(my $hsynth, "| gzip > $e->{synthesis}");
        close($hsynth);
    }

    unless ($nomediainfo) {
	print STDERR qq(link alternate locations of synthesis and hdlists\n) unless $nooutput;
	unlink $e->{hdlist2}, $e->{synthesis2};
	link $e->{hdlist}, $e->{hdlist2}
	    or print STDERR qq(link failed for "$e->{hdlist2}": $!\n);
	link $e->{synthesis}, $e->{synthesis2}
	    or print STDERR qq(link failed for "$e->{synthesis2}": $!\n);
    }

    unless ($nomd5sum) {
	print STDERR qq(generate media-specific MD5SUM in $e->{thismediainfo}\n) unless $nooutput;
	my $here = getcwd();
	chdir $e->{thismediainfo};
	my $md5sum = `/usr/bin/md5sum hdlist* synthesis*`;
	if (open my $md5sumfh, '>', $e->{md5sum}) {
	    print $md5sumfh $md5sum;
	    close $md5sumfh;
	} else {
	    print STDERR qq(Can't create "$e->{md5sum}": $!\n);
	}
	chdir $here;
    }
}

clean_cache();

if (grep { ! $_->{noneedrebuild} } @hdlists) {

    print STDERR "building base files\n" unless $nooutput;
    $urpm->build_base_files(
	depslist => $urpmfiles{depslist},
	provides => $urpmfiles{provides},
	compss   => $urpmfiles{compss},
    );

    if (-f $destinfodir . '/media.cfg') {
	if (! -f $destinfodir . '/hdlists' ||
	    ((stat($distrib->getfullpath(undef, 'infodir') . '/media.cfg'))[9] >
		(stat($destinfodir . '/hdlists'))[9])) {
	    print STDERR "Write hdlists file\n" unless $nooutput;
	    $distrib->write_hdlists($destinfodir . '/hdlists')
		or print STDERR "Can't write $destinfodir/hdlists file\n";
	}
    }

    #- safety cleaning
    unlink $urpmfiles{md5sum};
    unless ($nomd5sum) {
	my $here = getcwd();
	chdir $destinfodir;
	my $md5sum = `/usr/bin/md5sum hdlist* synthesis*`;
	if (open my $md5sumfh, '>', $urpmfiles{md5sum}) {
	    print $md5sumfh $md5sum;
	    close $md5sumfh;
	} else {
	    print STDERR qq(Can't create "$urpmfiles{md5sum}": $!\n);
	}
	chdir $here;
    }

    print STDERR "Building version file\n" unless $nooutput;
    $distrib->write_version($urpmfiles{version});
}

#- check if there are NOTFOUND in dependencies, check if they are in other media, warn the user.
if ($nooutput || !$nochkdep) {
    foreach (0 .. $#{$urpm->{depslist}}) {
        my $pkg = $urpm->{depslist}[$_];

        foreach (split " ", $urpm->{deps}[$_]) {
	    /NOTFOUND_(.*)/ or next;
	    print STDERR $pkg->fullname . " requires [$1] which\n";
	    if ($urpm->{provides}{$1}) {
	        print STDERR "  is available on packages not listed in this medium or previous medium:\n";
	        foreach (keys %{$urpm->{provides}{$1}}) {
		    my $dep_pkg = $urpm->{depslist}[$_];
		    print STDERR "    " . $dep_pkg->fullname . "\n";
	        }
	    } else {
	        print STDERR "  is not available in any medium listed\n";
	        if (/NOTFOUND_(\D*)(\d+[\.\-\d]*)?(.*)?\.so\./) {
		    my $re = (quotemeta $1) . '(\d+[\.\-\d]*)' . (!$2 && "?") . '\.so\.';
		    foreach (keys %{$urpm->{provides}}) {
		        /$re/ or next;
		        print STDERR "  but a similar provides is available as [$_], need rebuild ?\n";
		    }
	        }
            }
        }
    }
}

__END__

=head1 NAME

gendistrib - generates a mirror tree for a distribution

=head1 SYNOPSIS

    gendistrib [options] directory

=head1 OPTIONS

=over 4

=item --blind

Always rebuild indexes, without checking whether it's needed.

=item --compss file

Path of F<compss> file (defaults to F<media/media_info/compss>).

=item --depslist file

Path of F<depslist> file (defaults to F<media/media_info/depslist.ordered>).

=item --destdir dir

Create all new files in the specified directory. All subdirectories should
exist. This option is mostly useful for testing, or while using a read-only
repository.

=item --hdlists file

Path of the F<hdlists> file (defaults to F<media/media_info/hdlists>)

=item --headersdir dir

Put temporary files in this directory (defaults to TMPDIR).

=item --mediacfg file

Use the specified F<media.cfg> file (defaults to F<media/media_info/media.cfg>).

=item --nobadrpm

Don't abort when encountering bad rpms.

=item --nochkdep

Don't search for missing dependencies.

=item --noclean

Keep cache files.

=item --noemptymedia

Stop and abort if an empty media is found.

=item --nomd5sum

Don't generate MD5SUM files.

=item --nomediainfo

Don't create per-media F<media_info> subdirectories.

=item --provides file

Path of F<provides> file (defaults to F<media/media_info/provides>)

=item --skipmissingdir

If a media dir is missing, ignore it instead of aborting.

=item -s

Silent mode.

=back

=head1 DESCRIPTION

F<gendistrib> is a tool that helps to generate the structure of a Mandriva
RPM repository, compatible with Mandriva tools (F<urpmi>, F<rpmdrake>,
etc.)

=head2 General Structure of a Repository

A typical repository, under a root directory F</ROOT/>, has the following
structure:

    ROOT/ - media/
	    |- contrib/
	    |   `- media_info/
	    |- main/
	    |   `- media_info/
	    `- media_info/

In this example, we have two media, called I<main> and I<contrib>. The
RPMs packages are placed in the F<main> and F<contrib> subdirectories.
Repository metadata is contained in the top-level F<media_info> directory.
Per-media metadata are contained in the F<main/media_info> and
F<contrib/media_info> subdirectories.

=head2 Configuration of the distribution tree

Before using F<gendistrib>, you must create a file F<media_info/media.cfg>
to describe your repository. (An empty file will work, but this isn't
recommended.) The syntax of this file is reminiscent of F<.ini> files.

A first section C<[media_info]> contains global information about the
repository:

    [media_info]
    version=2006.0
    branch=Cooker
    arch=i586

Then, supply one section per media.

    [main]
    hdlist=hdlist_main.cz
    name=Main

Here, the C<hdlist> parameter specifies what will be the name of the
hdlist file in the top-level F<media_info> directory. C<name> is a human
readable label for the media.

=head2 Operation

F<genhdlist> should be passed the F<ROOT> directory as parameter. It will
then generate the hdlist and synthesis files and all other files needed
for proper repository operation.

F<genhdlist> will also verify any broken dependencies in your repository
and report them.

=head1 SEE ALSO

genhdlist(1)

=head1 COPYRIGHT

Copyright (C) 1999-2005 Mandrakesoft

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.

=cut