diff options
Diffstat (limited to 'modules/buildsystem')
70 files changed, 3267 insertions, 112 deletions
diff --git a/modules/buildsystem/files/Mageia.pm b/modules/buildsystem/files/Mageia.pm new file mode 100644 index 00000000..443f6cb7 --- /dev/null +++ b/modules/buildsystem/files/Mageia.pm @@ -0,0 +1,509 @@ +package Youri::Repository::Mageia; + +=head1 NAME + +Youri::Repository::Mageia - Mageia repository implementation + +=head1 DESCRIPTION + +This module implements Mageia repository + +=cut + +use warnings; +use strict; +use Carp; +use Memoize; +use File::Find 'find'; +use base qw/Youri::Repository/; +use MDV::Distribconf::Build; +use SVN::Client; +use Sys::Hostname; + +use constant { + PACKAGE_CLASS => 'Youri::Package::RPM::URPM', + PACKAGE_CHARSET => 'utf8' +}; + +memoize('_get_media_config'); + +my %translate_arch = ( + i386 => 'i586', + sparc64 => 'sparcv9', +); + +sub _init { + my $self = shift; + my %options = ( + noarch => 'i586', # noarch packages policy + src => 'i586', + install_root => '', + test => 0, # test mode + verbose => 0, # verbose mode + queue => '', + rejected => '', + @_ + ); + foreach my $var ('upload_state') { + $self->{"_$var"} = []; + foreach my $value (split ' ', $options{$var}) { + push @{$self->{"_$var"}}, $value + } + } + print "Initializing repository\n"; + foreach my $v ('rejected', 'svn', 'queue', 'noarch', 'install_root', 'upload_root', 'verbose') { + $self->{"_$v"} = $options{$v} + } + foreach my $target (@{$options{targets}}) { + $self->{$target} = []; + print "Adding $target ($options{$target}{arch})\n" if $self->{_verbose}; + foreach my $value (split ' ', $options{$target}{arch}) { + push @{$self->{_arch}{$target}}, $value; + push @{$self->{_extra_arches}}, $value + } + } + $self +} + +sub get_group_id { + my ($user) = @_; + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time); + $year+=1900; + $mon++; + my ($host) = hostname =~ /([^.]*)/; + sprintf "$year%02d%02d%02d%02d%02d.$user.$host.${$}_", $mon, $mday, $hour, $min, $sec; +} + +sub get_target_arch { + my ($self, $target) = $_; + return $self->{_arch}{$target} +} + +sub set_arch_changed { + my ($self, $target, $arch) = @_; + if ($arch eq 'noarch') { + $self->{_arch_changed}{$_} = 1 foreach @{$self->{_arch}{$target}} + } elsif ($arch eq 'src') { + $self->{_arch_changed} = $self->{_src} + } else { + $self->{_arch_changed}{$arch} = 1 + } +} + +sub get_arch_changed { + my ($self, $target) = @_; + return [ keys %{$self->{_arch_changed}} ] +} + +sub set_install_dir_changed { + my ($self, $install_dir) = @_; + $self->{_install_dir_changed}{$install_dir} = 1; +} + +sub get_install_dir_changed { + my ($self) = @_; + return [ keys %{$self->{_install_dir_changed}} ]; +} + +sub _get_media_config { + my ($self, $target) = @_; + my %media; + my $real_target = $target; + $real_target =~ s/_force//; + foreach my $arch (@{$self->{_arch}{$target}}) { + my $root = "$self->{_install_root}/$real_target/$arch"; + my $distrib = MDV::Distribconf::Build->new($root); + print "Getting media config from $root\n" if $self->{_verbose}; + $self->{distrib}{$arch} = $distrib; + $distrib->loadtree or die "$root does not seem to be a distribution tree\n"; + $distrib->parse_mediacfg; + foreach my $media ($distrib->listmedia) { + my $rpms = $distrib->getvalue($media, 'rpms'); + my $debug_for = $distrib->getvalue($media, 'debug_for'); + my $srpms = $distrib->getvalue($media, 'srpms'); + my $path = $distrib->getfullpath($media, 'path'); + if (!$rpms) { + if (-d $path) { + print "MEDIA defining $media in $path\n" if $self->{_verbose} > 1; + $media{$arch}{$media} = $path + } else { + print "ERROR $path does not exist for media $media on $arch\n" + } + } else { + my ($media) = split ' ', $rpms; + if (-d $path) { + print "MEDIA defining SOURCE media for $media in $path\n" if $self->{_verbose} > 1; + $media{src}{$media} = $path + } else { + print "ERROR $path does not exist for source media $media on $arch\n" + } + } + } + } + \%media +} + +sub get_package_class { + return PACKAGE_CLASS; +} + +sub get_package_charset { + return PACKAGE_CHARSET; +} + +sub get_upload_dir { + my ($self, $package, $target, $user_context, $app_context) = @_; + croak "Not a class method" unless ref $self; + my $arch = $package->get_arch(); + return + $self->{_upload_root} . + "/$self->{_queue}/$target/" . + _get_section($self, $package, $target, $user_context, $app_context) . + '/' . + ($user_context->{prefix} ? '' : get_group_id($user_context->{user})) +} + +sub get_install_path { + my ($self, $package, $target, $user_context, $app_context) = @_; + + return $self->_get_path($package, $target, $user_context, $app_context); +} + + +sub get_distribution_paths { + my ($self, $package, $target) = @_; + + return $self->_get_distribution_paths($package, $target); +} + +=head2 get_distribution_roots() + +Returns distribution roots (ie install_root + target + arch) +(it returns a list in case of noarch) + +=cut + +sub get_distribution_roots { + my ($self, $package, $target) = @_; + croak "Not a class method" unless ref $self; + + map { + $self->_get_dir($self->{_install_root}, $_); + } $self->_get_distribution_paths($package, $target); +} + +sub get_archive_path { + my ($self, $package, $target, $user_context, $app_context) = @_; + + return $self->_get_path($package, $target, $user_context, $app_context); +} + +sub get_reject_path { + my ($self, $package, $target, $user_context, $app_context) = @_; + + return $self->{_rejected}; +} + + +sub _get_path { + my ($self, $package, $target, $user_context, $app_context) = @_; + + my $section = $self->_get_section($package, $target, $user_context, $app_context); + my $arch = $app_context->{arch} || $package->get_arch(); + $arch = $translate_arch{$arch} || $arch; + if ($arch eq 'noarch') { + $arch = $self->{_noarch} + } elsif ($arch eq 'src') { + return "$target/SRPMS/$section" + } + "$target/$arch/media/$section" +} + +sub _get_distribution_paths { + my ($self, $package, $target) = @_; + + my $arch = $package->get_arch(); + $arch = $translate_arch{$arch} || $arch; + if ($arch eq 'noarch') { + map { "$target/$_" } $self->get_target_arches($target); + } elsif ($arch eq 'src') { + die "no way to get distribution path using a $arch package"; + } else { + "$target/$arch"; + } +} + +sub get_arch { + my ($self, $package, $target, $user_context, $app_context) = @_; + my $arch = $package->get_arch(); + $arch = $translate_arch{$arch} || $arch; + if ($arch eq 'noarch') { + $arch = $self->{_noarch} + } + $arch +} + +sub get_version_path { + my ($self, $package, $target, $user_context, $app_context) = @_; + + my $section = $self->_get_section($package, $target, $user_context, $app_context); + + return "$self->{_module}/$section"; +} + +=head2 get_replaced_packages($package, $target, $user_context, $app_context) + +Overrides parent method to add libified packages. + +=cut + +sub get_replaced_packages { + my ($self, $package, $target, $user_context, $app_context) = @_; + croak "Not a class method" unless ref $self; + + my @replaced_packages = + $self->SUPER::get_replaced_packages($package, $target, $user_context, $app_context); + + my $name = $package->get_name(); + + # kernel packages have the version in the name + # binary dkms built for old kernels have to be removed too + if ($name =~ /^kernel-([^\d]*-)?([\d.]*)-(.*)$/) { # "desktop", "2.6.28", "2mnb" + push(@replaced_packages, + map { PACKAGE_CLASS->new(file => $_) } + $self->get_files( + $self->{_install_root}, + $self->get_install_path($package, $target, $user_context, $app_context), + PACKAGE_CLASS->get_pattern( + '(kernel-' . $1 . '\d.*|.*-kernel-[\d.]*-' . $1 . '\d.*)', + undef, + undef, + $package->get_arch() + ), + ) + ); + } + + return @replaced_packages; + +} + +sub _get_main_section { + my ($self, $package, $target, $user_context, $app_context) = @_; + + my $section = $self->_get_section($package, $target, $user_context, $app_context); + my ($main_section) = $section =~ m,^([^/]+),; + $main_section +} + +sub _get_section { + my ($self, $package, $target, $user_context, $app_context) = @_; + + my $name = $package->get_name(); + my $cname = $package->get_canonical_name(); + my $version = $package->get_version(); + my $release = $package->get_release(); + my $section = $user_context->{section}; + my $media = $self->_get_media_config($target); + my $arch = $package->get_arch(); + my $file = $package->as_file(); + $file =~ s,/+,/,g; # unneeded? + # FIXME: use $self->get_arch() + $arch = $self->{_noarch} if $arch eq 'noarch'; + $arch = $translate_arch{$arch} || $arch; + + if (!$section) { + $section = $self->{packages}{$file}{section}; + print "Section undefined, repository says it is '$section' for '$file'\n" if $self->{_verbose}; + } + # FIXME: use debug_for info + if ($section && $section !~ m|debug/| && $package->is_debug()) { + $section = "debug/$section" + } + + # if have section already, check if it exists, and may return immediately + if ($section) { + print "Using requested section $section\n" if $self->{_verbose}; + if ($media->{$arch}{$section}) { + return $section + } else { + die "FATAL youri: unknown section $section for target $target for arch $arch\n" + } + } + # else, try to find section automatically + + # pattern for search of src package with specific version-release, + # should be searched first, because we prefer to find the precise + # section a package is already in + my $specific_source_pattern = PACKAGE_CLASS->get_pattern( + $cname, + $version, + $release, + 'src' + ); + + my $source_pattern = PACKAGE_CLASS->get_pattern( + $cname, + undef, + undef, + 'src' + ); + + # if a media has no source media configured, or if it is a debug + # package, we search in binary media + + # pattern for search when a binary media has no src media configured + my $specific_binary_pattern = PACKAGE_CLASS->get_pattern( + $name, + $version, + $release, + $arch + ); + + # last resort pattern: previous existing binary packages + my $binary_pattern = PACKAGE_CLASS->get_pattern( + $name, + undef, + undef, + $arch + ); + + # first try to find section for the specific version, as it is possibly already there; + # this is the case for when called in Youri::Submit::Action::Archive, to find the + # section the package got installed + print "Looking for package $name with version $version-$release\n" if $self->{_verbose}; + foreach my $m (keys %{$media->{$arch}}) { + print " .. section '$m' path '".$media->{$arch}{$m}."'\n" if $self->{_verbose}; + # - prefer source for non-debug packages, use binary if there is no source media configured + # - debug packages must be searched in binary medias, due to their + # src section != binary section; NOTE: should/need we search in + # src medias and add the 'debug_' prefix? + if (!$package->is_debug() && $media->{src}{$m}) { + next unless $self->get_files('', $media->{src}{$m}, $specific_source_pattern); + } else { + next unless $self->get_files('', $media->{$arch}{$m}, $specific_binary_pattern); + } + $section = $m; + last; + } + + # if still not found, try finding any version of the package in a + # /release subsection (safe default: /release is default for cooker, + # should be locked for released distros, and we don't risk wrongly + # choosing /backports, /testing, or /updates); + # this is the case for when called at submit, to find the section where + # the package already resides + if (!$section) { + # debug packages should be found by previous specific version search + # NOTE: as above, should/need we search here and add the 'debug_' prefix? + # ... probably... as at least mga-youri-submit-force will process debug packages + if ($package->is_debug() && $self->{_verbose}) { + print "Warning: debug package $name with version $version-$release not found.\n"; + } + + print "Warning: Looking for any section with a package $name of any version\n"; + foreach my $m (keys %{$media->{$arch}}) { + print " .. section '$m' path '".$media->{$arch}{$m}."'\n" if $self->{_verbose}; + # NOTE: !$package->is_debug() test is here to prevent when above FATAL error is removed + next if $m !~ /release/ || ($m =~ /debug/ && !$package->is_debug()); + # - prefer source + if ($media->{src}{$m}) { + next unless $self->get_files('', $media->{src}{$m}, $source_pattern); + } else { + next unless $self->get_files('', $media->{$arch}{$m}, $binary_pattern); + } + $section = $m; + last; + } + } + + # FIXME: doing this here is wrong; this way the caller can never know if + # a section was actually found or not; should return undef and let the + # caller set a default (Note: IIRC PLF|Zarb has this right, see there) -spuk + print STDERR "Warning: Can't guess destination: section missing, defaulting to core/release\n" unless $section; + $section ||= 'core/release'; + + # next time we don't need to search everything again + $self->{packages}{$file}{section} = $section; + + print "Section is '$section'.\n" if $self->{_verbose}; + + return $section; +} + +sub get_upload_newer_revisions { + my ($self, $package, $target, $user_context, $app_context) = @_; + croak "Not a class method" unless ref $self; + my $arch = $package->get_arch(); + my $name = $package->as_string(); + $name =~ s/^\@\d+://; + my $pattern = $self->get_package_class()->get_pattern($package->get_name(), undef, undef, $arch); + my $media = $self->_get_media_config($target); + my @packages; + foreach my $state (@{$self->{_upload_state}}) { + foreach my $m (keys %{$media->{$arch}}) { + next if defined($user_context->{section}) and $user_context->{section} ne $m; + my $path = "$self->{_upload_root}/$state/$target/$m"; + print "Looking for package $package revisions for $target in $path (pattern $pattern)\n" if $self->{_verbose}; + find( + sub { + s/\d{14}\.[^.]*\.[^.]*\.\d+_//; + s/^\@\d+://; + return if ! /^$pattern/; + return if /\.info$/; + print "Find $_\n" if $self->{_verbose} > 1; + push @packages, $File::Find::name if $package->check_ranges_compatibility("== $name", "< $_") + }, $path); + } + } + return + @packages; +} + +sub package_in_svn { + my ($self, $srpm_name) = @_; + my $ctx = new SVN::Client( + auth => [SVN::Client::get_simple_provider(), + SVN::Client::get_simple_prompt_provider(\&simple_prompt,2), + SVN::Client::get_username_provider()] + ); + + my $svn_entry = $ctx->ls("$self->{_svn}/$srpm_name", 'HEAD', 0); + if ($svn_entry) { + print "Package $srpm_name is in the SVN\n" if $self->{_verbose}; + return 1 + } +} + +sub get_svn_url { + my ($self) = @_; + $self->{_svn} +} + +sub reject { + my ($self, $package, $target, $user_context, $app_context) = @_; + croak "Not a class method" unless ref $self; + + +} + +sub get_archive_dir { + my ($self, $package, $target, $user_context, $app_context) = @_; + croak "Not a class method" unless ref $self; + + return + $self->{_archive_root} +} + + +=head1 COPYRIGHT AND LICENSE + +Copyright (C) 2002-2006, YOURI project +Copyright (C) 2006,2007,2009 Mandriva +Copyright (C) 2011 Nicolas Vigier, Michael Scherer, Pascal Terjan + +This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself. + +=cut + +1; diff --git a/modules/buildsystem/files/signbot/mga-signpackage b/modules/buildsystem/files/signbot/mga-signpackage new file mode 100755 index 00000000..199dbe0e --- /dev/null +++ b/modules/buildsystem/files/signbot/mga-signpackage @@ -0,0 +1,31 @@ +#!/usr/bin/perl -w + +use strict; +use warnings; +use RPM4::Sign; +use File::Spec; + +sub signpackage { + my ($file, $name, $path) = @_; + + # check if parent directory is writable + my $parent = (File::Spec->splitpath($file))[1]; + die "Unsignable package, parent directory is read-only" + unless -w $parent; + + my $sign = RPM4::Sign->new( + name => $name, + path => $path, + passphrase => '', + ); + + $sign->rpmssign($file) +} + +if (@ARGV != 3) { + exit 1; +} + +signpackage(@ARGV); +exit 0 + diff --git a/modules/buildsystem/files/signbot/sign-check-package b/modules/buildsystem/files/signbot/sign-check-package new file mode 100644 index 00000000..fc9704fd --- /dev/null +++ b/modules/buildsystem/files/signbot/sign-check-package @@ -0,0 +1,37 @@ +#!/bin/sh + +if [ $# != 3 ] ; then + echo "missing arguments" + echo "usage : $0 file key_number key_directory" + exit 1 +fi + +file="$1" +key="$2" +keydir="$3" + +tmpdir=`mktemp -d ${TMPDIR:-/tmp}/signbot-XXXXX` +tmpfile="$tmpdir/$(basename $file)" +cp -pf "$file" "$tmpfile" +rpm --delsign "$tmpfile" +/usr/local/bin/mga-signpackage "$tmpfile" "$key" "$keydir" +nbtry=0 +while rpmsign -Kv "$tmpfile" 2>&1 | grep BAD +do + nbtry=$(($nbtry + 1)) + if [ $nbtry -ge 30 ] + then + exit 1 + fi + + # Archive failed file for further analysis + mkdir -p "/tmp/failed-sign/" + failedfile="/tmp/failed-sign/$(basename "$file").$(date +%Y%m%d%H%M%S)" + cp -pf "$file" "$failedfile" + + cp -pf "$file" "$tmpfile" + rpm --delsign "$tmpfile" + /usr/local/bin/mga-signpackage "$tmpfile" "$key" "$keydir" +done +mv -f "$tmpfile" "$file" +rmdir "$tmpdir" diff --git a/modules/buildsystem/files/signbot/signbot-rpmmacros b/modules/buildsystem/files/signbot/signbot-rpmmacros new file mode 100644 index 00000000..aab7e389 --- /dev/null +++ b/modules/buildsystem/files/signbot/signbot-rpmmacros @@ -0,0 +1,3 @@ +%__gpg_sign_cmd %{__gpg} \ + gpg --batch --force-v3-sigs --no-verbose --no-armor --passphrase-fd 3 --no-secmem-warning \ + -u "%{_gpg_name}" -sbo %{__signature_filename} %{__plaintext_filename} diff --git a/modules/buildsystem/manifests/binrepo.pp b/modules/buildsystem/manifests/binrepo.pp new file mode 100644 index 00000000..5bf16b53 --- /dev/null +++ b/modules/buildsystem/manifests/binrepo.pp @@ -0,0 +1,48 @@ +class buildsystem::binrepo { + include buildsystem::var::binrepo + include buildsystem::var::groups + include sudo + + # upload-bin script uses the mailx command provided by nail +if versioncmp($::lsbdistrelease, '9') < 0 { + package { 'nail': + ensure => installed, + } +} else { + package { 's-nail': + ensure => installed, + } +} + + user { $buildsystem::var::binrepo::login: + home => $buildsystem::var::binrepo::homedir, + } + + file { [$buildsystem::var::binrepo::repodir, $buildsystem::var::binrepo::uploadinfosdir]: + ensure => directory, + owner => $buildsystem::var::binrepo::login, + } + + mga_common::local_script { + 'upload-bin': + content => template('buildsystem/binrepo/upload-bin'); + 'wrapper.upload-bin': + content => template('buildsystem/binrepo/wrapper.upload-bin'); + } + + sudo::sudoers_config { 'binrepo': + content => template('buildsystem/binrepo/sudoers.binrepo') + } + + apache::vhost::base { $buildsystem::var::binrepo::hostname: + location => $buildsystem::var::binrepo::repodir, + content => template('buildsystem/binrepo/vhost_binrepo.conf'), + } + + apache::vhost::base { "ssl_${buildsystem::var::binrepo::hostname}": + use_ssl => true, + vhost => $buildsystem::var::binrepo::hostname, + location => $buildsystem::var::binrepo::repodir, + content => template('buildsystem/binrepo/vhost_binrepo.conf'), + } +} diff --git a/modules/buildsystem/manifests/buildnode.pp b/modules/buildsystem/manifests/buildnode.pp new file mode 100644 index 00000000..1573c093 --- /dev/null +++ b/modules/buildsystem/manifests/buildnode.pp @@ -0,0 +1,12 @@ +class buildsystem::buildnode { + include buildsystem::iurt + include buildsystem::var::scheduler + include buildsystem::var::iurt + include buildsystem::sshkeys + + sshkeys::set_authorized_keys { 'iurt-allow-scheduler': + keyname => $buildsystem::var::scheduler::login, + home => $buildsystem::var::iurt::homedir, + user => $buildsystem::var::iurt::login, + } +} diff --git a/modules/buildsystem/manifests/create_upload_dir.rb b/modules/buildsystem/manifests/create_upload_dir.rb new file mode 100644 index 00000000..8023ab5d --- /dev/null +++ b/modules/buildsystem/manifests/create_upload_dir.rb @@ -0,0 +1,28 @@ +hostclass "buildsystem::create_upload_dir" do + states = ["todo","done","failure","queue","rejected"] + owner = scope.lookupvar('buildsystem::var::scheduler::login') + group = owner + uploads_dir = scope.lookupvar('buildsystem::var::scheduler::homedir') + '/uploads' + + file uploads_dir, :ensure => 'directory', :owner => owner, :group => group + + for st in states do + file [uploads_dir, st].join('/'), :ensure => 'directory', :owner => owner, :group => group + + scope.lookupvar('buildsystem::var::distros::distros').each{|rel, distro| + file [uploads_dir, st, rel].join('/'), :ensure => 'directory', :owner => owner, :group => group + medias = distro['medias'] + medias.each{|media, m| + file [uploads_dir, st, rel, media].join('/'), :ensure => 'directory', :owner => owner, :group => group + + for repo in m['repos'].keys do + if st == 'done' + file [uploads_dir, st, rel, media, repo].join('/'), :ensure => 'directory', :owner => owner, :group => group, :mode => 0775 + else + file [uploads_dir, st, rel, media, repo].join('/'), :ensure => 'directory', :owner => owner, :group => group + end + end + } + } + end +end diff --git a/modules/buildsystem/manifests/distros.rb b/modules/buildsystem/manifests/distros.rb new file mode 100644 index 00000000..a298c0a8 --- /dev/null +++ b/modules/buildsystem/manifests/distros.rb @@ -0,0 +1,97 @@ +hostclass "buildsystem::distros" do + mirror_user = 'root' + schedbot_user = scope.lookupvar('buildsystem::var::scheduler::login') + bootstrap_reporoot = scope.lookupvar('buildsystem::var::repository::bootstrap_reporoot') + scope.lookupvar('buildsystem::var::distros::distros').each{|rel, distro| + file [ bootstrap_reporoot, rel ].join('/'), :ensure => 'directory', + :owner => mirror_user, :group => mirror_user + for arch in distro['arch'] do + # As ruby dsl cannot use defined resources, we have to use a + # workaround with 'find_resource_type' as described in this + # puppet issue: http://projects.puppetlabs.com/issues/11912 + scope.find_resource_type 'buildsystem::media_cfg' + media_cfg_args = { + :distro_name => rel, + :arch => arch, + } + if distro['tmpl_media.cfg'] != nil + media_cfg_args['templatefile'] = distro['tmpl_media.cfg'] + end + if ! distro['no_media_cfg_update'] + create_resource 'buildsystem::media_cfg', + [ rel, ' ', arch ].join('/'), media_cfg_args + end + file [ bootstrap_reporoot, rel, arch ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + mediadir = [ bootstrap_reporoot, rel, arch, 'media' ].join('/') + file mediadir, :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, 'media_info' ].join('/'), :ensure => 'directory', + :owner => schedbot_user, :group => schedbot_user + file [ mediadir, 'debug' ].join('/'), :ensure => 'directory', + :owner => schedbot_user, :group => schedbot_user + distro['medias'].each{|media, m| + file [ mediadir, media ].join('/'), :ensure => 'directory', + :owner => schedbot_user, :group => schedbot_user + file [ mediadir, 'debug', media ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + for repo in m['repos'].keys do + file [ mediadir, media, repo ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, media, repo, 'media_info' ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, media, repo, 'repodata' ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, 'debug', media, repo ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, 'debug', media, repo, 'media_info' ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, 'debug', media, repo, 'repodata' ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + end + } + if distro['based_on'] != nil + distro['based_on'].each{|bdistroname, medias| + file [ mediadir, bdistroname ].join('/'), + :ensure => 'directory', :owner => mirror_user, + :group => mirror_user + medias.each{|medianame, media| + mdir = [ mediadir, bdistroname, medianame ].join('/') + file mdir, :ensure => 'directory', + :owner => mirror_user, :group => mirror_user + for reponame in media + file [ mdir, reponame ].join('/'), + :ensure => 'link', + :target => [ + '../../../../..', bdistroname, arch, + 'media', medianame, reponame ].join('/'), + :owner => mirror_user, :group => mirror_user + end + } + } + end + end + # SRPMS + srpmsdir = [ bootstrap_reporoot, rel, 'SRPMS' ].join('/') + file srpmsdir, + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + distro['medias'].each{|media, m| + file [ srpmsdir, media ].join('/'), :ensure => 'directory', + :owner => schedbot_user, :group => schedbot_user + for repo in m['repos'].keys do + file [ srpmsdir, media, repo ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + end + } + } +end diff --git a/modules/buildsystem/manifests/gatherer.pp b/modules/buildsystem/manifests/gatherer.pp new file mode 100644 index 00000000..eebfd97e --- /dev/null +++ b/modules/buildsystem/manifests/gatherer.pp @@ -0,0 +1,5 @@ +class buildsystem::gatherer { + # emi is in main iurt rpm, should be moved out + include iurt::packages + include iurt::upload +} diff --git a/modules/buildsystem/manifests/init.pp b/modules/buildsystem/manifests/init.pp index e78468f2..f15b5dbf 100644 --- a/modules/buildsystem/manifests/init.pp +++ b/modules/buildsystem/manifests/init.pp @@ -1,83 +1,2 @@ class buildsystem { - - class base { - $build_login = "iurt" - $build_home_dir = "/home/iurt/" - - include ssh::auth - ssh::auth::key { $build_login: } # declare a key for build bot: RSA, 2048 bits - } - - class mainnode inherits base { - include iurtuser - ssh::auth::server { $build_login: } - - package { "task-bs-cluster-main": - ensure => "installed" - } - } - - class buildnode inherits base { - include iurt - } - - class scheduler { - # ulri - } - - class dispatcher { - # emi - } - - class repsys { - package { 'repsys': - - } - - - } - - class iurtuser { - group {"$build_login": - ensure => present, - } - - user {"$build_login": - ensure => present, - comment => "System user used to run build bots", - managehome => true, - gid => $build_login, - shell => "/bin/bash", - } - } - - class iurt { - include sudo - include iurtuser - ssh::auth::client { $build_login: } - - # build node common settings - # we could have the following skip list to use less space: - # '/(drakx-installer-binaries|drakx-installer-advertising|gfxboot|drakx-installer-stage2|mandriva-theme)/' - $package_list = ['task-bs-cluster-chroot', 'iurt'] - package { $package_list: - ensure => installed; - } - - file { "$build_home_dir/.iurt.cauldron.conf": - ensure => present, - owner => $build_login, - group => $build_login, - mode => 644, - content => template("buildsystem/iurt.cauldron.conf") - } - - file { "/etc/sudoers.d/iurt": - ensure => present, - owner => root, - group => root, - mode => 440, - content => template("buildsystem/sudoers.iurt") - } - } } diff --git a/modules/buildsystem/manifests/iurt.pp b/modules/buildsystem/manifests/iurt.pp new file mode 100644 index 00000000..231c5373 --- /dev/null +++ b/modules/buildsystem/manifests/iurt.pp @@ -0,0 +1,26 @@ +class buildsystem::iurt { + include sudo + include buildsystem::iurt::user + include buildsystem::iurt::packages + include buildsystem::var::iurt + include buildsystem::var::distros + + # remove old build directory + tidy { "${buildsystem::var::iurt::homedir}/iurt": + age => '8w', + recurse => true, + matches => ['[0-9][0-9].*\..*\..*\.[0-9]*','log','*.rpm','*.log','*.mga[0-9]+'], + rmdirs => true, + } + + file { '/etc/iurt/build': + ensure => directory, + } + + $distros_list = hash_keys($buildsystem::var::distros::distros) + buildsystem::iurt::config { $distros_list: } + + sudo::sudoers_config { 'iurt': + content => template('buildsystem/sudoers.iurt') + } +} diff --git a/modules/buildsystem/manifests/iurt/config.pp b/modules/buildsystem/manifests/iurt/config.pp new file mode 100644 index 00000000..02f5be63 --- /dev/null +++ b/modules/buildsystem/manifests/iurt/config.pp @@ -0,0 +1,50 @@ +define buildsystem::iurt::config() { + include buildsystem::var::iurt + include buildsystem::var::webstatus + include buildsystem::var::repository + $distribution = $name + # TODO rename the variable too in template + $build_login = $buildsystem::var::iurt::login + + $build_timeout = { + 'default' => 36000, + 'atlas' => 57600, + 'blender' => 57600, + 'chromium-browser-stable' => 172800, + 'clang' => 172800, + 'cross-gcc' => 115200, + 'gcc' => 115200, + 'itk' => 115200, + 'java-1.8.0-openjdk' => 172800, + 'java-17-openjdk' => 172800, + 'java-21-openjdk' => 172800, + 'java-latest-openjdk' => 172800, + 'kernel' => 115200, + 'libreoffice' => 432000, + 'llvm' => 115200, + 'llvm17-suite' => 115200, + 'llvm19-suite' => 115200, + 'openfoam' => 115200, + 'paraview' => 115200, + 'qgis' => 57600, + 'qtwebengine5' => 115200, + 'qtwebengine6' => 172800, + 'rust' => 180000, + 'salome' => 57600, + 'vtk' => 57600, + 'webkit' => 57600, + 'webkit2' => 115200, + 'wrapitk' => 115200, + 'rocm-llvm' => 70000, + } + + $allow_network_access = [ + 'libguestfs', # Needs access to the configured mirrors + ] + + file { "/etc/iurt/build/${distribution}.conf": + owner => $build_login, + group => $build_login, + content => template("buildsystem/iurt.conf") + } +} diff --git a/modules/buildsystem/manifests/iurt/packages.pp b/modules/buildsystem/manifests/iurt/packages.pp new file mode 100644 index 00000000..e814b7c2 --- /dev/null +++ b/modules/buildsystem/manifests/iurt/packages.pp @@ -0,0 +1,3 @@ +class buildsystem::iurt::packages { + package { 'iurt': } +} diff --git a/modules/buildsystem/manifests/iurt/upload.pp b/modules/buildsystem/manifests/iurt/upload.pp new file mode 100644 index 00000000..5417d36e --- /dev/null +++ b/modules/buildsystem/manifests/iurt/upload.pp @@ -0,0 +1,16 @@ +class buildsystem::iurt::upload { + include buildsystem::var::iurt + include buildsystem::var::webstatus + include buildsystem::var::repository + file { '/etc/iurt/upload.conf': + require => File['/etc/iurt'], + content => template('buildsystem/upload.conf'), + notify => Exec['check iurt config'], + } + + exec { 'check iurt config': + refreshonly => true, + command => 'perl -cw /etc/iurt/upload.conf', + logoutput => 'on_failure', + } +} diff --git a/modules/buildsystem/manifests/iurt/user.pp b/modules/buildsystem/manifests/iurt/user.pp new file mode 100644 index 00000000..a93ac7e7 --- /dev/null +++ b/modules/buildsystem/manifests/iurt/user.pp @@ -0,0 +1,11 @@ +class buildsystem::iurt::user { + include buildsystem::var::iurt + + buildsystem::sshuser { $buildsystem::var::iurt::login: + homedir => $buildsystem::var::iurt::homedir, + } + + file { '/etc/iurt': + ensure => directory, + } +} diff --git a/modules/buildsystem/manifests/mainnode.pp b/modules/buildsystem/manifests/mainnode.pp new file mode 100644 index 00000000..01de764f --- /dev/null +++ b/modules/buildsystem/manifests/mainnode.pp @@ -0,0 +1,23 @@ +class buildsystem::mainnode { + include buildsystem::var::repository + include buildsystem::var::scheduler + include buildsystem::var::distros + include buildsystem::iurt::user + include buildsystem::scheduler + include buildsystem::gatherer + include buildsystem::mgarepo + include buildsystem::signbot + include buildsystem::youri_submit + include buildsystem::sshkeys + include buildsystem::distros + + sshkeys::set_client_key_pair { $buildsystem::var::scheduler::login: + home => $buildsystem::var::scheduler::homedir, + user => $buildsystem::var::scheduler::login, + } + sshkeys::set_authorized_keys { 'scheduler-allow-scheduler': + keyname => $buildsystem::var::scheduler::login, + home => $buildsystem::var::scheduler::homedir, + user => $buildsystem::var::scheduler::login, + } +} diff --git a/modules/buildsystem/manifests/maintdb.pp b/modules/buildsystem/manifests/maintdb.pp new file mode 100644 index 00000000..5a961b63 --- /dev/null +++ b/modules/buildsystem/manifests/maintdb.pp @@ -0,0 +1,58 @@ +class buildsystem::maintdb { + include buildsystem::var::maintdb + include buildsystem::var::groups + include buildsystem::var::webstatus + include sudo + + user { $buildsystem::var::maintdb::login: + home => $buildsystem::var::maintdb::homedir, + } + + file { [$buildsystem::var::maintdb::homedir,$buildsystem::var::maintdb::dbdir]: + ensure => directory, + owner => $buildsystem::var::maintdb::login, + group => $buildsystem::var::maintdb::login, + mode => '0711', + require => User[$buildsystem::var::maintdb::login], + } + + file { $buildsystem::var::maintdb::binpath: + mode => '0755', + content => template('buildsystem/maintdb/maintdb.bin') + } + + mga_common::local_script { 'wrapper.maintdb': + content => template('buildsystem/maintdb/wrapper.maintdb') + } + + sudo::sudoers_config { 'maintdb': + content => template('buildsystem/maintdb/sudoers.maintdb') + } + + file { [$buildsystem::var::maintdb::dump, + "${buildsystem::var::maintdb::dump}.new", + $buildsystem::var::maintdb::unmaintained, + "${buildsystem::var::maintdb::unmaintained}.new"]: + owner => $buildsystem::var::maintdb::login, + require => File["${buildsystem::var::webstatus::location}/data"], + } + + cron { 'update maintdb export': + user => $buildsystem::var::maintdb::login, + command => "${buildsystem::var::maintdb::binpath} root get > ${buildsystem::var::maintdb::dump}.new; cp -f ${buildsystem::var::maintdb::dump}.new ${buildsystem::var::maintdb::dump}; grep ' nobody\$' ${buildsystem::var::maintdb::dump} | sed 's/ nobody\$//' > ${buildsystem::var::maintdb::unmaintained}.new; cp -f ${buildsystem::var::maintdb::unmaintained}.new ${buildsystem::var::maintdb::unmaintained}", + minute => '*/30', + require => User[$buildsystem::var::maintdb::login], + } + + apache::vhost::base { $buildsystem::var::maintdb::hostname: + location => $buildsystem::var::maintdb::dbdir, + content => template('buildsystem/maintdb/vhost_maintdb.conf'), + } + + apache::vhost::base { "ssl_${buildsystem::var::maintdb::hostname}": + use_ssl => true, + vhost => $buildsystem::var::maintdb::hostname, + location => $buildsystem::var::maintdb::dbdir, + content => template('buildsystem/maintdb/vhost_maintdb.conf'), + } +} diff --git a/modules/buildsystem/manifests/media_cfg.pp b/modules/buildsystem/manifests/media_cfg.pp new file mode 100644 index 00000000..77fcc8fd --- /dev/null +++ b/modules/buildsystem/manifests/media_cfg.pp @@ -0,0 +1,11 @@ +define buildsystem::media_cfg($distro_name, $arch, $templatefile = 'buildsystem/media.cfg') { + include buildsystem::var::repository + include buildsystem::var::scheduler + include buildsystem::repository + + file { "${buildsystem::var::repository::bootstrap_reporoot}/${distro_name}/${arch}/media/media_info/media.cfg": + owner => $buildsystem::var::scheduler::login, + group => $buildsystem::var::scheduler::login, + content => template($templatefile), + } +} diff --git a/modules/buildsystem/manifests/mgarepo.pp b/modules/buildsystem/manifests/mgarepo.pp new file mode 100644 index 00000000..14e11e1a --- /dev/null +++ b/modules/buildsystem/manifests/mgarepo.pp @@ -0,0 +1,36 @@ +class buildsystem::mgarepo { + include buildsystem::var::scheduler + include buildsystem::var::distros + include buildsystem::var::groups + include buildsystem::var::binrepo + include buildsystem::create_upload_dir + $sched_login = $buildsystem::var::scheduler::login + $sched_home_dir = $buildsystem::var::scheduler::homedir + + package { ['mgarepo','rpm-build']: } + + file { '/etc/mgarepo.conf': + content => template('buildsystem/mgarepo.conf'), + } + + file { "${sched_home_dir}/repsys": + ensure => 'directory', + owner => $sched_login, + require => File[$sched_home_dir], + } + + file { ["${sched_home_dir}/repsys/tmp", "${sched_home_dir}/repsys/srpms"]: + ensure => 'directory', + owner => $sched_login, + group => $buildsystem::var::groups::packagers, + mode => '1775', + require => File["${sched_home_dir}/repsys"], + } + + # FIXME: disabled temporarily as upload dir is a symlink to /var/lib/repsys/uploads + #file { "${sched_home_dir}/uploads": + # ensure => "directory", + # owner => $sched_login, + # require => File[$sched_home_dir], + #} +} diff --git a/modules/buildsystem/manifests/release.pp b/modules/buildsystem/manifests/release.pp new file mode 100644 index 00000000..d9feac8e --- /dev/null +++ b/modules/buildsystem/manifests/release.pp @@ -0,0 +1,5 @@ +class buildsystem::release { + git::snapshot { '/root/release': + source => "git://git.${::domain}/software/infrastructure/release", + } +} diff --git a/modules/buildsystem/manifests/repoctl.pp b/modules/buildsystem/manifests/repoctl.pp new file mode 100644 index 00000000..8d44e52c --- /dev/null +++ b/modules/buildsystem/manifests/repoctl.pp @@ -0,0 +1,11 @@ +class buildsystem::repoctl { + include buildsystem::var::distros + include buildsystem::var::repository + + package{ 'repoctl': } + + file { '/etc/repoctl.conf': + content => template('buildsystem/repoctl.conf'), + require => Package['repoctl'], + } +} diff --git a/modules/buildsystem/manifests/repository.pp b/modules/buildsystem/manifests/repository.pp new file mode 100644 index 00000000..dda90eb2 --- /dev/null +++ b/modules/buildsystem/manifests/repository.pp @@ -0,0 +1,11 @@ +class buildsystem::repository { + include buildsystem::var::repository + file { [ $buildsystem::var::repository::bootstrap_root, + $buildsystem::var::repository::bootstrap_reporoot ] : + ensure => directory, + } + + apache::vhost::other_app { $buildsystem::var::repository::hostname: + vhost_file => 'buildsystem/vhost_repository.conf', + } +} diff --git a/modules/buildsystem/manifests/rpmlint.pp b/modules/buildsystem/manifests/rpmlint.pp new file mode 100644 index 00000000..388d0bee --- /dev/null +++ b/modules/buildsystem/manifests/rpmlint.pp @@ -0,0 +1,3 @@ +class buildsystem::rpmlint { + package { 'rpmlint': } +} diff --git a/modules/buildsystem/manifests/scheduler.pp b/modules/buildsystem/manifests/scheduler.pp new file mode 100644 index 00000000..53b248fc --- /dev/null +++ b/modules/buildsystem/manifests/scheduler.pp @@ -0,0 +1,57 @@ +class buildsystem::scheduler { + # until ulri is split from main iurt rpm + include buildsystem::iurt::packages + include buildsystem::iurt::upload + include buildsystem::var::scheduler + + $login = $buildsystem::var::scheduler::login + $homedir = $buildsystem::var::scheduler::homedir + $logdir = $buildsystem::var::scheduler::logdir + + buildsystem::sshuser { $login: + homedir => $homedir, + } + + file { $logdir: + ensure => directory, + mode => '0755', + owner => $login, + } + + cron { 'dispatch jobs': + user => $login, + command => "EMI_LOG_FILE=${logdir}/emi.log ULRI_LOG_FILE=${logdir}/ulri.log ulri; EMI_LOG_FILE=${logdir}/emi.log emi", + minute => '*', + } + + if ($buildsystem::var::scheduler::clean_uploads_logs_age != 0) { + cron { 'clean uploads logs': + user => $login, + # Delete old upload logs + command => sprintf("/usr/bin/find %s/uploads -ignore_readdir_race -xdev -depth -type f -ctime +%d -delete", shellquote($homedir), shellquote($buildsystem::var::scheduler::clean_uploads_logs_age)), + hour => '*/4', + minute => '51', + } + cron { 'clean uploads dirs': + user => $login, + # Remove old empty uploads directories. This will take several + # passes (over several weeks) to delete a directory hierarchy + # because it is looking at ctime instead of mtime, which resets + # every time a file/directory underneath it is deleted. + # Directories don't take much space, so this shouldn't be a + # real issue. + command => sprintf("/usr/bin/find %s/uploads -ignore_readdir_race -mindepth 5 -xdev -depth -type d -ctime +%d -empty -delete", shellquote($homedir), shellquote($buildsystem::var::scheduler::clean_uploads_logs_age)), + hour => '*/4', + minute => '53', + } + } + if ($buildsystem::var::scheduler::clean_uploads_packages_age != 0) { + cron { 'clean uploads packages': + user => $login, + # Delete old upload RPMs + command => sprintf("/usr/bin/find %s/uploads -ignore_readdir_race -xdev -depth -type f -name '*.rpm' -ctime +%d -delete", shellquote($homedir), shellquote($buildsystem::var::scheduler::clean_uploads_packages_age)), + hour => '*/4', + minute => '52', + } + } +} diff --git a/modules/buildsystem/manifests/signbot.pp b/modules/buildsystem/manifests/signbot.pp new file mode 100644 index 00000000..60c7c318 --- /dev/null +++ b/modules/buildsystem/manifests/signbot.pp @@ -0,0 +1,31 @@ +class buildsystem::signbot { + include buildsystem::var::scheduler + include buildsystem::var::signbot + $sched_login = $buildsystem::var::scheduler::login + + sshuser { $buildsystem::var::signbot::login: + homedir => $buildsystem::var::signbot::home_dir, + groups => [$sched_login], + } + + gnupg::keys{ 'packages': + email => $buildsystem::var::signbot::keyemail, + key_name => $buildsystem::var::signbot::keyname, + login => $buildsystem::var::signbot::login, + batchdir => "${buildsystem::var::signbot::home_dir}/batches", + keydir => $buildsystem::var::signbot::sign_keydir, + } + + sudo::sudoers_config { 'signpackage': + content => template('buildsystem/signbot/sudoers.signpackage') + } + + file { "${home_dir}/.rpmmacros": + source => 'puppet:///modules/buildsystem/signbot/signbot-rpmmacros', + } + + mga_common::local_script { + 'sign-check-package': source => 'puppet:///modules/buildsystem/signbot/sign-check-package'; + 'mga-signpackage': source => 'puppet:///modules/buildsystem/signbot/mga-signpackage'; + } +} diff --git a/modules/buildsystem/manifests/sshkeys.pp b/modules/buildsystem/manifests/sshkeys.pp new file mode 100644 index 00000000..5a1b2900 --- /dev/null +++ b/modules/buildsystem/manifests/sshkeys.pp @@ -0,0 +1,5 @@ +class buildsystem::sshkeys { + include buildsystem::var::scheduler + + sshkeys::create_key { $buildsystem::var::scheduler::login: } +} diff --git a/modules/buildsystem/manifests/sshuser.pp b/modules/buildsystem/manifests/sshuser.pp new file mode 100644 index 00000000..5cad97ad --- /dev/null +++ b/modules/buildsystem/manifests/sshuser.pp @@ -0,0 +1,36 @@ +# $groups: array of secondary groups (only local groups, no ldap) +define buildsystem::sshuser($homedir, $comment = undef, $groups = []) { + group { $name: } + + user { $name: + comment => $comment, + managehome => true, + home => $homedir, + gid => $name, + groups => $groups, + shell => '/bin/bash', + notify => Exec["unlock ${name}"], + require => Group[$title], + } + + # set password to * to unlock the account but forbid login through login + exec { "unlock ${name}": + command => "usermod -p '*' ${name}", + refreshonly => true, + } + + file { $homedir: + ensure => directory, + owner => $name, + group => $name, + require => User[$name], + } + + file { "${homedir}/.ssh": + ensure => directory, + mode => '0600', + owner => $name, + group => $name, + require => File[$homedir], + } +} diff --git a/modules/buildsystem/manifests/var/binrepo.pp b/modules/buildsystem/manifests/var/binrepo.pp new file mode 100644 index 00000000..1431ed25 --- /dev/null +++ b/modules/buildsystem/manifests/var/binrepo.pp @@ -0,0 +1,15 @@ +# $uploadmail_from: +# from who will be sent the binrepo upload email notifications +# $uploadmail_to: +# where binrepo email notifications are sent +class buildsystem::var::binrepo( + $hostname = "binrepo.${::domain}", + $login = 'binrepo', + $homedir = '/var/lib/binrepo', + $uploadmail_from, + $uploadmail_to +) { + $repodir = "${homedir}/data" + $uploadinfosdir = "${homedir}/infos" + $uploadbinpath = '/usr/local/bin/upload-bin' +} diff --git a/modules/buildsystem/manifests/var/distros.pp b/modules/buildsystem/manifests/var/distros.pp new file mode 100644 index 00000000..9e45e2c2 --- /dev/null +++ b/modules/buildsystem/manifests/var/distros.pp @@ -0,0 +1,126 @@ +# $default_distro: +# the name of the default distribution +# $repo_allow_from_ips: +# $repo_allow_from_domains: +# list of IP or domains allowed to access the repository. If you don't want to +# filter allowed IPs, don't those values. +# $distros: +# a hash variable containing distributions information indexed by +# distribution name. Each distribution is itself an hash containing +# the following infos: +# { +# # the 'cauldron' distribution +# 'cauldron' => { +# # list of arch supported by 'cauldron' +# 'arch' => [ 'i586', 'x86_64' ], +# # Set this if you don't want media.cfg to be generated +# 'no_media_cfg_update' => true, +# 'medias' => { +# # the 'core' media +# 'core' => { +# 'repos' => { +# # the 'release' repo in the 'core' media +# 'release' => { +# 'media_type' => [ 'release' ], +# 'noauto' => '1', +# # the 'release' repo should be listed first in media.cfg +# 'order' => 0, +# }, +# # the 'updates' repo +# 'updates' => { +# 'media_type' => [ 'updates' ], +# 'noauto' => '1', +# # the 'updates' repo requires the 'release' repo +# 'requires' => [ 'release' ], +# # the 'updates' repo should be listed after 'release' in media.cfg +# 'order' => 1, +# }, +# }, +# # media_type for media.cfg +# 'media_type' => [ 'official', 'free' ], +# # if noauto is set to '1' either in medias or repos, +# # the option will be added to media.cfg +# 'noauto' => '1', +# # list 'core' first in media.cfg +# 'order' => 0, +# }, +# # the 'non-free' media +# 'non-free' => { +# 'repos' => { +# ... +# }, +# 'media_type' => [ 'official', 'non-free' ], +# # the 'non-free' media requires the 'core' media +# 'requires' => [ 'core' ], +# # list 'non-free' second +# 'order' => 1, +# } +# }, +# # the list of media used by iurt to build the chroots +# 'base_medias' => [ 'core/release' ], +# # optionally, a media.cfg template file can be specified, if +# # the default one should not be used +# 'tmpl_media.cfg' => 'buildsystem/something', +# # branch is Devel or Official. Used in media.cfg. +# 'branch' => 'Devel', +# # Version of the distribution +# 'version' => '3', +# # SVN Urls allowed to submit +# 'submit_allowed' => 'svn://svn.something/svn/packages/cauldron', +# # rpm macros to set when build source package +# 'macros' => { +# 'distsuffix' => '.mga', +# 'distribution' => 'Mageia', +# 'vendor' => 'Mageia.Org', +# }, +# # set this if the distro is not mirrored. This is used to add +# # an Alias in the vhost. +# 'no_mirror' => true, +# Optionally, the distribution can be based on the repos from an other +# distribution. In this example we're saying that the distribution is +# based on 2/core/release and 2/core/updates. +# 'based_on' => { +# '2' => { +# 'core' => [ 'release', 'updates' ], +# }, +# }, +# 'youri' => { +# # Configuration for youri-upload +# 'upload' => { +# # list of enabled checks, actions and posts +# 'targets' => { +# 'checks' => [ +# ... +# ], +# 'actions' => [ +# ... +# ], +# 'posts' => [ +# ... +# ], +# }, +# 'checks' => { +# # rpmlint checks options +# 'rpmlint' => { +# 'config' => '/usr/share/rpmlint/config', +# 'path' => ''/usr/bin/rpmlint', +# }, +# }, +# # options for actions +# 'actions' => { +# ... +# }, +# }, +# # Configuration for youri-todo +# 'todo' => { +# ... +# }, +# }, +# }, +# } +class buildsystem::var::distros( + $default_distro, + $repo_allow_from_ips, + $repo_allow_from_domains, + $distros, +) { } diff --git a/modules/buildsystem/manifests/var/groups.pp b/modules/buildsystem/manifests/var/groups.pp new file mode 100644 index 00000000..c0b2c917 --- /dev/null +++ b/modules/buildsystem/manifests/var/groups.pp @@ -0,0 +1,9 @@ +# $packagers: +# name of packagers group, who should be allowed to submit packages +# $packagers_committers: +# name of group of users who are allowed to commit on packages +class buildsystem::var::groups( + $packagers, + $packagers_committers +) { +} diff --git a/modules/buildsystem/manifests/var/iurt.pp b/modules/buildsystem/manifests/var/iurt.pp new file mode 100644 index 00000000..fb65a160 --- /dev/null +++ b/modules/buildsystem/manifests/var/iurt.pp @@ -0,0 +1,5 @@ +class buildsystem::var::iurt( + $login = 'iurt', + $homedir = '/home/iurt', + $timeout_multiplier = 1, +) { } diff --git a/modules/buildsystem/manifests/var/maintdb.pp b/modules/buildsystem/manifests/var/maintdb.pp new file mode 100644 index 00000000..e0079e40 --- /dev/null +++ b/modules/buildsystem/manifests/var/maintdb.pp @@ -0,0 +1,11 @@ +class buildsystem::var::maintdb( + $hostname = "maintdb.${::domain}", + $login = 'maintdb', + $homedir = '/var/lib/maintdb' +) { + include buildsystem::var::webstatus + $dbdir = "${homedir}/db" + $binpath = '/usr/local/sbin/maintdb' + $dump = "${buildsystem::var::webstatus::location}/data/maintdb.txt" + $unmaintained = "${buildsystem::var::webstatus::location}/data/unmaintained.txt" +} diff --git a/modules/buildsystem/manifests/var/mgarepo.pp b/modules/buildsystem/manifests/var/mgarepo.pp new file mode 100644 index 00000000..9099c7ee --- /dev/null +++ b/modules/buildsystem/manifests/var/mgarepo.pp @@ -0,0 +1,22 @@ +# $submit_host: +# hostname used to submit packages +# $svn_hostname: +# hostname of the svn server used for packages +# $svn_root_packages: +# svn root url of the svn repository for packages +# $svn_root_packages_ssh: +# svn+ssh root url of the svn repository for packages +# $oldurl: +# svn url where the import logs of the rpm are stored +# $conf: +# $conf{'global'} is a has table of values used in mgarepo.conf in +# the [global] section +class buildsystem::var::mgarepo( + $submit_host, + $svn_hostname, + $svn_root_packages, + $svn_root_packages_ssh, + $oldurl, + $conf +) { +} diff --git a/modules/buildsystem/manifests/var/repository.pp b/modules/buildsystem/manifests/var/repository.pp new file mode 100644 index 00000000..0ea1058c --- /dev/null +++ b/modules/buildsystem/manifests/var/repository.pp @@ -0,0 +1,9 @@ +class buildsystem::var::repository( + $hostname = "repository.${::domain}", + $bootstrap_root = '/distrib/bootstrap', + $mirror_root = '/distrib/mirror', + $distribdir = 'distrib' +) { + $bootstrap_reporoot = "${bootstrap_root}/${distribdir}" + $mirror_reporoot = "${mirror_root}/${distribdir}" +} diff --git a/modules/buildsystem/manifests/var/scheduler.pp b/modules/buildsystem/manifests/var/scheduler.pp new file mode 100644 index 00000000..b431594c --- /dev/null +++ b/modules/buildsystem/manifests/var/scheduler.pp @@ -0,0 +1,31 @@ +# $admin_mail: +# the email address from which the build failure notifications +# will be sent +# $pkg_uphost: +# hostname of the server where submitted packages are uploaded +# $build_nodes: +# a hash containing available build nodes indexed by architecture +# $build_nodes_aliases: +# a hash containing build nodes indexed by their alias +# $build_src_node: +# hostname of the server building the initial src.rpm +# $clean_uploads_logs_age: +# old logs are cleaned when they are older than some amount of days. +# You can define this amount of time using this variable. Set it to +# 14 for two weeks, 2 for two days, or 0 if you don't want to +# clean old logs at all +# $clean_uploads_packages_age: +# same as $clean_uploads_logs_age but for old RPMs +class buildsystem::var::scheduler( + $admin_mail = "root@${::domain}", + $pkg_uphost = "pkgsubmit.${::domain}", + $build_nodes, + $build_nodes_aliases = {}, + $build_src_node, + $clean_uploads_logs_age = 14, + $clean_uploads_packages_age = 7 +){ + $login = 'schedbot' + $homedir = "/var/lib/${login}" + $logdir = "/var/log/${login}" +} diff --git a/modules/buildsystem/manifests/var/signbot.pp b/modules/buildsystem/manifests/var/signbot.pp new file mode 100644 index 00000000..7d92a324 --- /dev/null +++ b/modules/buildsystem/manifests/var/signbot.pp @@ -0,0 +1,15 @@ +# $keyid: +# the key id of the gnupg key used to sign packages +# $keyemail: +# email address of the key used to sign packages +# $keyname: +# name of the key used to sign packages +class buildsystem::var::signbot( + $keyid, + $keyemail, + $keyname +) { + $login = 'signbot' + $home_dir = "/var/lib/${login}" + $sign_keydir = "${home_dir}/keys" +} diff --git a/modules/buildsystem/manifests/var/webstatus.pp b/modules/buildsystem/manifests/var/webstatus.pp new file mode 100644 index 00000000..21f8d59f --- /dev/null +++ b/modules/buildsystem/manifests/var/webstatus.pp @@ -0,0 +1,25 @@ +# $git_url: +# git url where the sources of webstatus are located +# $hostname: +# vhost name of the webstatus page +# $location: +# path of the directory where the webstatus files are located +# $package_commit_url: +# url to view a commit on a package. %d is replaced by the commit id. +# $max_modified: +# how much history should we display, in days +# $theme_name: +# name of the webstatus theme +# $themes_dir: +# path of the directory where the themes are located. If you want +# to use a theme not included in webstatus, you need to change this. +class buildsystem::var::webstatus( + $git_url = "git://git.${::domain}/web/pkgsubmit", + $hostname = "pkgsubmit.${::domain}", + $location = '/var/www/bs', + $package_commit_url, + $max_modified = '2', + $theme_name = 'mageia', + $themes_dir = '/var/www/bs/themes/' +) { +} diff --git a/modules/buildsystem/manifests/var/youri.pp b/modules/buildsystem/manifests/var/youri.pp new file mode 100644 index 00000000..f20b6c7b --- /dev/null +++ b/modules/buildsystem/manifests/var/youri.pp @@ -0,0 +1,401 @@ +# The youri configuration files are created using information from 3 +# different hash variables : +# - the $youri_conf_default variable defined in this class, containing +# the default configuration for youri. It contains the repository +# configuration, and the definitions of the checks, actions and posts. +# - the $youri_conf parameter passed to this class. The values defined +# in this hash override the values defined in the default configuration. +# - for each distribution defined in the hash variable $distros from +# var::buildsystem::distros the hash defined in index 'youri' contains +# some distro specific options for youri checks, actions or posts. It +# also contains for each distribution the list of active checks, +# actions and posts. +# +# Each of those variables contain the configuration for youri submit-todo +# (in index 'todo') and youri submit-upload (in index 'upload') +# +# +# Parameters : +# $tmpl_youri_upload_conf: +# template file for youri submit-upload.conf +# $tmpl_youri_todo_conf: +# template file for youri submit-todo.conf +# $packages_archivedir: +# the directory where youri will archive old packages when they are +# replaced by a new version +# $youri_conf: +# a hash containing the youri configuration +class buildsystem::var::youri( + $tmpl_youri_upload_conf = 'buildsystem/youri/submit.conf', + $tmpl_youri_todo_conf = 'buildsystem/youri/submit.conf', + $packages_archivedir, + $youri_conf = {} +) { + include buildsystem::var::repository + include buildsystem::var::mgarepo + include buildsystem::var::distros + include buildsystem::var::signbot + include buildsystem::var::scheduler + + $check_tag = { 'class' => 'Youri::Submit::Check::Tag', } + $check_recency = { 'class' => 'Youri::Submit::Check::Recency', } + $check_queue_recency = { 'class' => 'Youri::Submit::Check::Queue_recency', } + $check_host = { + 'class' => 'Youri::Submit::Check::Host', + 'options' => { + 'host_file' => '/etc/youri/host.conf', + }, + } + $check_rpmlint = { 'class' => 'Youri::Submit::Check::Rpmlint', } + $check_acl = { + 'class' => 'Youri::Submit::Check::ACL', + 'options' => { + 'acl_file' => '/etc/youri/acl.conf', + }, + } + $check_source = { 'class' => 'Youri::Submit::Check::Source', } + $check_version = { + 'class' => 'Youri::Submit::Check::Version', + 'options' => {}, + } + + $youri_conf_default = { + 'upload' => { + 'repository' => { + 'class' => 'Youri::Repository::Mageia', + 'options' => { + 'install_root' => $buildsystem::var::repository::bootstrap_reporoot, + 'upload_root' => '$home/uploads/', + 'archive_root' => $packages_archivedir, + 'upload_state' => 'queue', + 'queue' => 'queue', + 'noarch' => 'x86_64', + 'svn' => "${buildsystem::var::mgarepo::svn_root_packages_ssh}/${buildsystem::var::distros::default_distro}", + }, + }, + 'checks' => { + 'tag' => $check_tag, + 'recency' => $check_recency, + 'queue_recency' => $check_queue_recency, + 'host' => $check_host, + 'section' => { + 'class' => 'Youri::Submit::Check::Section', + }, + 'rpmlint' => $check_rpmlint, + 'svn' => { + 'class' => 'Youri::Submit::Check::SVN', + }, + 'acl' => $check_acl, + 'history' => { + 'class' => 'Youri::Submit::Check::History', + }, + 'source' => $check_source, + 'precedence' => { + 'class' => 'Youri::Submit::Check::Precedence', + 'options' => { + 'target' => $buildsystem::var::distros::default_distro, + }, + }, + 'version' => $check_version, + }, + 'actions' => { + 'install' => { + 'class' => 'Youri::Submit::Action::Install', + }, + 'markrelease' => { + 'class' => 'Youri::Submit::Action::Markrelease', + }, + 'link' => { + 'class' => 'Youri::Submit::Action::Link', + }, + 'archive' => { + 'class' => 'Youri::Submit::Action::Archive', + }, + 'clean' => { + 'class' => 'Youri::Submit::Action::Clean', + }, + 'sign' => { + 'class' => 'Youri::Submit::Action::Sign', + 'options' => { + 'signuser' => $buildsystem::var::signbot::login, + 'path' => $buildsystem::var::signbot::sign_keydir, + 'name' => $buildsystem::var::signbot::keyid, + 'signscript' => '/usr/local/bin/sign-check-package', + }, + }, + 'unpack_gfxboot_theme' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'mageia-gfxboot-theme', + 'source_subdir' => '/usr/share/gfxboot/themes/Mageia/install/', + 'dest_directory' => 'isolinux', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_meta_task' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'meta-task', + 'source_subdir' => '/usr/share/meta-task', + 'dest_directory' => 'media/media_info', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_images' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-images', + 'source_subdir' => '/usr/lib*/drakx-installer-images', + 'dest_directory' => '.', + 'preclean_directory' => 'install/images/alternatives', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_images_nonfree' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-images-nonfree', + 'source_subdir' => '/usr/lib*/drakx-installer-images', + 'dest_directory' => '.', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_stage2' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-stage2', + 'source_subdir' => '/usr/lib*/drakx-installer-stage2', + 'dest_directory' => '.', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_advertising' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-advertising', + 'source_subdir' => '/usr/share/drakx-installer-advertising', + 'dest_directory' => '.', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_rescue' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-rescue', + 'source_subdir' => '/usr/lib*/drakx-installer-rescue', + 'dest_directory' => 'install/stage2', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_release_notes' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'mageia-release-common', + 'source_subdir' => '/usr/share/doc/mageia-release-common', + 'grep_files' => 'release-notes.*', + 'dest_directory' => '.', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_syslinux' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'syslinux', + 'source_subdir' => '/usr/lib/syslinux/', + 'grep_files' => '\\(hdt\\|ifcpu\\|ldlinux\\|libcom32\\|libgpl\\|libmenu\\|libutil\\).c32', + 'dest_directory' => 'isolinux', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_pci_usb_ids' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'ldetect-lst', + 'source_subdir' => '/usr/share/', + 'grep_files' => '\\(pci\\|usb\\).ids', + 'dest_directory' => 'isolinux', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'mail' => { + 'class' => 'Youri::Submit::Action::Mail', + 'options' => { + 'mta' => '/usr/sbin/sendmail', + }, + }, + 'maintdb' => { + 'class' => 'Youri::Submit::Action::UpdateMaintDb', + }, + 'rebuild' => { + 'class' => 'Youri::Submit::Action::RebuildPackage', + 'options' => { + 'rules' => { + 'drakx-installer-binaries' => ['drakx-installer-images'], + 'drakx-kbd-mouse-x11' => ['drakx-installer-stage2'], + 'drakx-net' => ['drakx-installer-stage2'], + 'kernel-desktop-latest' => ['drakx-installer-images', 'kmod-virtualbox', 'kmod-xtables-addons'], + 'kernel-desktop586-latest' => ['drakx-installer-images', 'kmod-virtualbox', 'kmod-xtables-addons'], + 'kernel-server-latest' => ['kmod-virtualbox', 'kmod-xtables-addons'], + 'ldetect-lst' => ['drakx-installer-stage2'], + 'meta-task' => ['drakx-installer-stage2'], + 'perl' => ['drakx-installer-stage2'], + 'perl-URPM' => ['drakx-installer-stage2'], + 'rpm' => ['drakx-installer-stage2'], + 'rpm-mageia-setup' => ['drakx-installer-stage2'], + 'urpmi' => ['drakx-installer-stage2'], + }, + }, + }, + }, + 'posts' => { + 'genhdlist2' => { + 'class' => 'Youri::Submit::Post::Genhdlist2', + 'options' => { + 'command' => '/usr/bin/genhdlist2 --xml-info-filter ".lzma:xz -T4" --synthesis-filter ".cz:xz -7 -T8"', + }, + }, + 'genhdlist2_zstd' => { + 'class' => 'Youri::Submit::Post::Genhdlist2', + 'options' => { + 'command' => '/usr/bin/genhdlist2 --xml-info-filter ".lzma:xz -T4" --synthesis-filter ".cz:zstd -19 -T8"', + }, + }, + 'createrepo_mga6' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10', + }, + }, + 'createrepo_mga7' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10 --zck --zck-dict-dir /usr/share/mageia-repo-zdicts/mga7/', + }, + }, + 'createrepo_mga8' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10 --zck --zck-dict-dir /usr/share/mageia-repo-zdicts/mga7/', + }, + }, + 'createrepo_mga9' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10 --zck --zck-dict-dir /usr/share/mageia-repo-zdicts/mga7/', + }, + }, + 'createrepo_cauldron' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10', + }, + }, + 'appstream_mga6' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-6-modifyrepo.sh', + }, + }, + 'appstream_mga7' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-7-modifyrepo.sh', + }, + }, + 'appstream_mga8' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-8-modifyrepo.sh', + }, + }, + 'appstream_mga9' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-9-modifyrepo.sh', + }, + }, + 'appstream_cauldron' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-cauldron-modifyrepo.sh', + }, + }, + 'clean_rpmsrate' => { + 'class' => 'Youri::Submit::Post::CleanRpmsrate', + }, + 'mirror' => { + 'class' => 'Youri::Submit::Post::Mirror', + 'options' => { + 'destination' => $buildsystem::var::repository::mirror_reporoot, + }, + }, + }, + }, + 'todo' => { + 'repository' => { + 'class' => 'Youri::Repository::Mageia', + 'options' => { + 'install_root' => $buildsystem::var::repository::bootstrap_reporoot, + 'upload_root' => '$home/uploads/', + 'upload_state' => 'todo done queue', + 'queue' => 'todo', + 'noarch' => 'x86_64', + 'svn' => "${buildsystem::var::mgarepo::svn_root_packages_ssh}/${buildsystem::var::distros::default_distro}", + }, + }, + 'checks' => { + 'tag' => $check_tag, + 'recency' => $check_recency, + 'queue_recency' => $check_queue_recency, + 'host' => $check_host, + 'rpmlint' => $check_rpmlint, + 'acl' => $check_acl, + 'source' => $check_source, + 'version' => $check_version, + 'deps' => { + 'class' => 'Youri::Submit::Check::Deps', + }, + }, + 'actions' => { + 'send' => { + 'class' => 'Youri::Submit::Action::Send', + 'options' => { + 'user' => $buildsystem::var::scheduler::login, + 'keep_svn_release' => 'yes', + 'uphost' => $buildsystem::var::scheduler::pkg_uphost, + 'root' => '$home/uploads', + 'ssh_key' => '$home/.ssh/id_rsa', + }, + }, + 'dependencies' => { + 'class' => 'Youri::Submit::Action::Dependencies', + 'options' => { + 'user' => $buildsystem::var::scheduler::login, + 'uphost' => $buildsystem::var::scheduler::pkg_uphost, + 'root' => '$home/uploads', + 'ssh_key' => '$home/.ssh/id_rsa', + }, + }, + 'rpminfo' => { + 'class' => 'Youri::Submit::Action::Rpminfo', + 'options' => { + 'user' => $buildsystem::var::scheduler::login, + 'uphost' => $buildsystem::var::scheduler::pkg_uphost, + 'root' => '$home/uploads', + 'ssh_key' => '$home/.ssh/id_rsa', + }, + }, + 'ulri' => { + 'class' => 'Youri::Submit::Action::Ulri', + 'options' => { + 'user' => $buildsystem::var::scheduler::login, + 'uphost' => $buildsystem::var::scheduler::pkg_uphost, + 'ssh_key' => '$home/.ssh/id_rsa', + }, + }, + }, + 'posts' => { + }, + }, + } +} diff --git a/modules/buildsystem/manifests/webstatus.pp b/modules/buildsystem/manifests/webstatus.pp new file mode 100644 index 00000000..49346dbc --- /dev/null +++ b/modules/buildsystem/manifests/webstatus.pp @@ -0,0 +1,44 @@ +class buildsystem::webstatus { + include buildsystem::var::webstatus + include buildsystem::var::scheduler + include apache::mod::php + + file { [ $buildsystem::var::webstatus::location, "${buildsystem::var::webstatus::location}/data" ]: + ensure => directory, + } + + $vhost = $buildsystem::var::webstatus::hostname + apache::vhost::base { $vhost: + aliases => { + '/uploads' => "${buildsystem::var::scheduler::homedir}/uploads", + '/autobuild/cauldron/x86_64/core/log/status.core.log' => "${buildsystem::var::webstatus::location}/autobuild/broken.php", + '/themes' => $buildsystem::var::webstatus::themes_dir, + }, + location => $buildsystem::var::webstatus::location, + content => template('buildsystem/vhost_webstatus.conf'), + } + + apache::vhost::base { "ssl_${vhost}": + vhost => $vhost, + use_ssl => true, + aliases => { + '/uploads' => "${buildsystem::var::scheduler::homedir}/uploads", + '/autobuild/cauldron/x86_64/core/log/status.core.log' => "${buildsystem::var::webstatus::location}/autobuild/broken.php", + '/themes' => $buildsystem::var::webstatus::themes_dir, + }, + location => $buildsystem::var::webstatus::location, + content => template('buildsystem/vhost_webstatus.conf'), + } + + git::snapshot { $buildsystem::var::webstatus::location: + source => $buildsystem::var::webstatus::git_url, + } + + file { '/etc/bs-webstatus.conf': + ensure => present, + content => template('buildsystem/bs-webstatus.conf'), + mode => '0644', + owner => root, + group => root, + } +} diff --git a/modules/buildsystem/manifests/youri_submit.pp b/modules/buildsystem/manifests/youri_submit.pp new file mode 100644 index 00000000..6b4d7dc2 --- /dev/null +++ b/modules/buildsystem/manifests/youri_submit.pp @@ -0,0 +1,83 @@ +class buildsystem::youri_submit { + include sudo + include buildsystem::rpmlint + include buildsystem::repository + include buildsystem::var::scheduler + include buildsystem::var::youri + + mga_common::local_script { + 'mga-youri-submit': + content => template('buildsystem/mga-youri-submit'); + 'mga-youri-submit.wrapper': + content => template('buildsystem/mga-youri-submit.wrapper'); + 'submit_package': + content => template('buildsystem/submit_package.pl'); + 'mga-clean-distrib': + content => template('buildsystem/cleaner.rb'); + } + + sudo::sudoers_config { 'mga-youri-submit': + content => template('buildsystem/sudoers.youri') + } + $release_managers = group_members('mga-release_managers') + # ordering is automatic : + # https://docs.puppetlabs.com/learning/ordering.html#autorequire + file { + '/etc/youri/': + ensure => 'directory'; + '/etc/youri/acl.conf': + content => template('buildsystem/youri/acl.conf'); + '/etc/youri/host.conf': + content => template('buildsystem/youri/host.conf'); + } + + buildsystem::youri_submit_conf{ 'upload': + tmpl_file => $buildsystem::var::youri::tmpl_youri_upload_conf, + } + buildsystem::youri_submit_conf{ 'todo': + tmpl_file => $buildsystem::var::youri::tmpl_youri_todo_conf, + } + + cron { 'Archive orphan packages from cauldron': + command => "/usr/local/bin/mga-clean-distrib --auto -v cauldron -d ${buildsystem::var::youri::packages_archivedir} -l ${buildsystem::var::scheduler::homedir}/tmp/upload", + hour => 5, + minute => 30, + user => $buildsystem::var::scheduler::login, + } + + file { $buildsystem::var::youri::packages_archivedir: + ensure => 'directory', + owner => $buildsystem::var::scheduler::login, + require => File[$buildsystem::var::scheduler::homedir], + } + + tidy { $buildsystem::var::youri::packages_archivedir: + type => 'ctime', + recurse => true, + age => '1w', + matches => '*.rpm', + } + + include mga_common::var::perl + file { [ "${mga_common::var::perl::site_perl_dir}/Youri", + "${mga_common::var::perl::site_perl_dir}/Youri/Repository"]: + ensure => directory, + mode => '0755', + owner => root, + group => root, + } + file { "${mga_common::var::perl::site_perl_dir}/Youri/Repository/Mageia.pm": + source => 'puppet:///modules/buildsystem/Mageia.pm', + } + + $package_list= ['perl-SVN', + 'mdv-distrib-tools', + 'perl-Youri-Media', + 'perl-Youri-Package', + 'perl-Youri-Repository', + 'perl-Youri-Utils', + 'perl-Youri-Config', + 'mga-youri-submit'] + + package { $package_list: } +} diff --git a/modules/buildsystem/manifests/youri_submit_conf.pp b/modules/buildsystem/manifests/youri_submit_conf.pp new file mode 100644 index 00000000..28b911d9 --- /dev/null +++ b/modules/buildsystem/manifests/youri_submit_conf.pp @@ -0,0 +1,6 @@ +define buildsystem::youri_submit_conf($tmpl_file) { + $conf_name = $name + file { "/etc/youri/submit-${conf_name}.conf": + content => template($tmpl_file), + } +} diff --git a/modules/buildsystem/templates/binrepo/sudoers.binrepo b/modules/buildsystem/templates/binrepo/sudoers.binrepo new file mode 100644 index 00000000..c20810cf --- /dev/null +++ b/modules/buildsystem/templates/binrepo/sudoers.binrepo @@ -0,0 +1 @@ +%<%= scope.lookupvar('buildsystem::var::groups::packagers_committers') %> ALL =(<%= scope.lookupvar('buildsystem::var::binrepo::login') %>) NOPASSWD: <%= scope.lookupvar('buildsystem::var::binrepo::uploadbinpath') %> diff --git a/modules/buildsystem/templates/binrepo/upload-bin b/modules/buildsystem/templates/binrepo/upload-bin new file mode 100755 index 00000000..7cad5838 --- /dev/null +++ b/modules/buildsystem/templates/binrepo/upload-bin @@ -0,0 +1,32 @@ +#!/bin/sh +set -e + +binrepodir=<%= scope.lookupvar('buildsystem::var::binrepo::repodir') %> +uploadinfosdir=<%= scope.lookupvar('buildsystem::var::binrepo::uploadinfosdir') %> +tmpfile=$(mktemp) +mail_from="<%= scope.lookupvar('buildsystem::var::binrepo::uploadmail_from') %>" +mail_dest="<%= scope.lookupvar('buildsystem::var::binrepo::uploadmail_to') %>" + +test $# = 2 || exit 3 +username="$1" +comment="$2" + +/bin/cat > "$tmpfile" +sha1sum=$(/usr/bin/sha1sum "$tmpfile" | sed 's/ .*$//') +test -n "$sha1sum" +if [ -f "$binrepodir/$sha1sum" ] +then + echo "File $sha1sum already exists." >&2 + /bin/rm -f "$tmpfile" + exit 2 +fi +/bin/mv "$tmpfile" "$binrepodir/$sha1sum" +/bin/chmod 644 "$binrepodir/$sha1sum" +echo "$username:$comment" > "$uploadinfosdir/$sha1sum" +echo "User $username uploaded file $sha1sum: $comment" + +echo "User $username uploaded file $sha1sum: $comment" | \ + /usr/bin/mailx -s "New file uploaded: $sha1sum - $comment" -S "from=$username <$mail_from>" "$mail_dest" + +exit 0 + diff --git a/modules/buildsystem/templates/binrepo/vhost_binrepo.conf b/modules/buildsystem/templates/binrepo/vhost_binrepo.conf new file mode 100644 index 00000000..f411c07a --- /dev/null +++ b/modules/buildsystem/templates/binrepo/vhost_binrepo.conf @@ -0,0 +1,3 @@ +<Directory <%= scope.lookupvar('buildsystem::var::binrepo::repodir') %>> + Options None +</Directory> diff --git a/modules/buildsystem/templates/binrepo/wrapper.upload-bin b/modules/buildsystem/templates/binrepo/wrapper.upload-bin new file mode 100644 index 00000000..3def84a0 --- /dev/null +++ b/modules/buildsystem/templates/binrepo/wrapper.upload-bin @@ -0,0 +1,26 @@ +#!/bin/sh + +binrepouser="<%= scope.lookupvar('buildsystem::var::binrepo::login') %>" +uploadbinpath="<%= scope.lookupvar('buildsystem::var::binrepo::uploadbinpath') %>" +packagerscommittersgroup="<%= scope.lookupvar('buildsystem::var::groups::packagers_committers') %>" + +function isingroup() +{ + grp="$1" + for group in `groups` + do if [ "$grp" = "$group" ] + then + return 0 + fi + done + return 1 +} + +if ! isingroup "$packagerscommittersgroup" +then + echo "You are not in $packagerscommittersgroup group." + exit 1 +fi + +sudo -u "$binrepouser" "$uploadbinpath" $(whoami) $@ + diff --git a/modules/buildsystem/templates/bs-webstatus.conf b/modules/buildsystem/templates/bs-webstatus.conf new file mode 100644 index 00000000..9f37a990 --- /dev/null +++ b/modules/buildsystem/templates/bs-webstatus.conf @@ -0,0 +1,32 @@ +<?php + +/** Where is the current app located. */ +$g_webapp_dir = '<%= scope.lookupvar('buildsystem::var::webstatus::location') %>'; + +/** Full system path where packages are uploaded. */ +$upload_dir = '<%= scope.lookupvar('buildsystem::var::scheduler::homedir') %>/uploads'; + +/** How long a history should we keep, in days. */ +$max_modified = <%= scope.lookupvar('buildsystem::var::webstatus::max_modified') %>; + +/** How many nodes are available. */ +$g_nodes_count = 2; + +/** html > body > h1 title */ +$title = 'Build system status'; + +/** Should crawlers index this page or not? meta[robots] tag.*/ +$robots = 'index,nofollow,nosnippet,noarchive'; + +/** */ +$g_root_url = 'https://<%= scope.lookupvar('buildsystem::var::webstatus::hostname') %>/'; + +/** URL to view a package svn revision. %d is replaced by the revision */ +$package_commit_url = '<%= scope.lookupvar('buildsystem::var::webstatus::package_commit_url') %>'; + +/** name of the theme */ +$theme_name = '<%= scope.lookupvar('buildsystem::var::webstatus::theme_name') %>'; + +/** themes directory */ +$themes_dir = '<%= scope.lookupvar('buildsystem::var::webstatus::themes_dir') %>'; + diff --git a/modules/buildsystem/templates/cleaner.rb b/modules/buildsystem/templates/cleaner.rb new file mode 100755 index 00000000..fa0d08ca --- /dev/null +++ b/modules/buildsystem/templates/cleaner.rb @@ -0,0 +1,235 @@ +#!/usr/bin/ruby + +def usage + puts "Usage: #{$0} [options]" + puts "Moves obsolete packages" + puts + puts "-h, --help show this help" + puts "-a, --archs <arch1>,<arch2>,... list of architectures to clean" + puts "-a, --auto do not ask confirmation" + puts "-p, --base <path> base path to the repository" + puts "-m, --media <media1>,<media2>,... list of media to clean (default: core/release,tainted/release,nonfree/release)" + puts "-d, --destination <path> path to the old packages storage" + puts "-v, --version <version> version to clean (default: cauldron)" +end + +require 'fileutils' +require 'getoptlong' +require 'readline' + +def process + opts = GetoptLong.new( + [ '--help', '-h', GetoptLong::NO_ARGUMENT ], + [ '--archs', '-a', GetoptLong::REQUIRED_ARGUMENT ], + [ '--auto', '-A', GetoptLong::NO_ARGUMENT ], + [ '--base', '-p', GetoptLong::REQUIRED_ARGUMENT ], + [ '--media', '-m', GetoptLong::REQUIRED_ARGUMENT ], + [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ], + [ '--version', '-v', GetoptLong::REQUIRED_ARGUMENT ], + [ '--lockfile', '-l', GetoptLong::REQUIRED_ARGUMENT ], + ) + + base_path = "<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %>/distrib" + archs = [ "x86_64", "i686", "aarch64", "armv7hl" ] + medias = ["core/release", "tainted/release", "nonfree/release"] + old_path = "<%= scope.lookupvar('buildsystem::var::youri::packages_archivedir') %>" + version = "cauldron" + auto = false + lockfile = nil + + opts.each do |opt, arg| + case opt + when '--help' + usage + exit 0 + when '--destination' + old_path = arg + when '--media' + medias = arg.split(",") + when '--archs' + archs = arg.split(",") + when '--auto' + auto = true + when '--base' + base_path = arg + when '--version' + version = arg + when '--lockfile' + lockfile = arg + end + end + + take_upload_lock(lockfile) if lockfile + + medias.each{|media| + src_path = "#{base_path}/#{version}/SRPMS/#{media}" + + $used_srcs = {} + $old_srcs = {} + $srcs = {} + $srcages = {} + $noarch = {} + + # Get a list of all src.rpm and their build time + `urpmf --synthesis "#{src_path}/media_info/synthesis.hdlist.cz" --qf '%filename:%buildtime:%buildarchs' "."`.each_line{|l| + l2 = l.split(':') + filename = l2[0] + buildtime = l2[1].to_i + buildarch = l2[2].rstrip + name = name_from_filename(filename) + if $srcages[name] then + if buildtime < $srcages[name][1] then + # This src.rpm is older, ignore it and store it in the list to be deleted + $old_srcs[filename] = true + next + else + # This src.rpm has an older version, ignore that version and store it in the list to be deleted + old_filename = $srcages[name][0] + $old_srcs[old_filename] = true + $srcs.delete(old_filename) + end + end + $srcages[name] = [ filename, buildtime ] + $srcs[filename] = true + $noarch[name] = true if buildarch == 'noarch' + } + archs.each{|arch| + bin_path = "#{base_path}/#{version}/#{arch}/media/#{media}" + debug_path = bin_path.sub("/media/", "/media/debug/") + old_packages = check_binaries(arch, $srcs, $srcages, src_path, bin_path, $used_srcs) + old_debug_packages = check_binaries(arch, $srcs, {}, src_path, debug_path, nil) + move_packages(bin_path, old_path, old_packages, auto) + move_packages(debug_path, old_path, old_debug_packages, auto) + } + $used_srcs.keys.each{|s| $srcs.delete(s)} + + move_packages(src_path, old_path, $srcs.keys + $old_srcs.keys, auto) + } +end + +def take_upload_lock(path) + start_time = Time.new + has_lock = false + at_exit { + if File.exists?(path) + if File.readlines(path)[0].to_i == Process.pid + File.delete(path) + end + end + } + until has_lock + while File.exists?(path) + if Time.new - start_time > 2*3600.0 + puts "Could not acquire upload lock for more than 2h, giving up" + end + sleep(5) + end + File.write(path, Process.pid) + if File.readlines(path)[0].to_i == Process.pid + has_lock = true + end + end +end + +def move_packages(src, dst, list, auto) + list.reject!{|f| !File.exist?(src + "/" + f)} + return if list.empty? + list.each{|b| + puts b + } + puts "The #{list.length} listed packages will be moved from #{src} to #{dst}." + line = Readline::readline('Are you sure [Yn]? ') unless auto + if auto || line =~ /^y?$/i + list.each{|s| + oldfile = src + "/" + s + newfile = dst + "/" + s + next unless File.exist?(oldfile) + if (File.exist?(newfile)) + File.unlink(oldfile) + else + FileUtils.mv(oldfile, newfile) + end + } + end +end + +# For each binary media: +# - Check if we have the src.rpm (else the binary package is obsolete) +# * If we don't have the src.rpm, check if we have a newer version +# - If there is a new version: +# * check if this architecture has packages from it to avoid deleting armv7hl packages before the new one get rebuilt +# * check if the new version is old enough to allow rebuilding everything (7d?) +# - Mark used src.rpm (if one is never marked, the src.rpm is obsolete) + +def packages(path) + `urpmf --synthesis "#{path}/media_info/synthesis.hdlist.cz" --qf '%sourcerpm:%filename:%buildtime' ":"`.each_line{|l| + l2 = l.split(':') + sourcerpm = l2[0] + filename = l2[1] + buildtime = l2[2].to_i + yield(sourcerpm, filename, buildtime) + } +end + +def name_from_filename(filename) + filename.sub(/-[^-]*-[^-]*$/, '') +end + +def arch_wanted(src, arch) + exclusive_arch = `rpmquery -p #{src} --qf '[%{EXCLUSIVEARCH} ]'`.rstrip + if exclusive_arch != "" then + if !exclusive_arch.split(/ /).include?(arch) then + return false + end + end + exclude_arch = `rpmquery -p #{src} --qf '[%{EXCLUDEARCH} ]'`.rstrip + if exclude_arch != "" then + if exclude_arch.split(/ /).include?(arch) then + return false + end + end + + return true +end + +def check_binaries(arch, srcs, srcages, src_path, path, used_srcs) + used_here_srcs = {} + all_versions = {} + packages(path) {|src, filename, buildtime| + used_srcs[src] = true if used_srcs != nil + if filename =~ /noarch.rpm$/ then + # We need to mark the src.rpm present on this arch only for full noarch packages + used_here_srcs[src] = true if $noarch[name_from_filename(src)] + else + used_here_srcs[src] = true + end + name = name_from_filename(filename) + if all_versions[name] then + all_versions[name] << src + else + all_versions[name] = [src] + end + } + old_binaries = [] + packages(path) {|src, filename, buildtime| + if ! srcs[src] then + srcname = name_from_filename(src) + if srcages[srcname] then + # The src.rpm is gone but there is a different version of it + latestsrc = srcages[srcname][0] + # Only delete old binaries after 7d or if there is a new version + name = name_from_filename(filename) + next unless (srcages[srcname][1] < Time.now.to_i - 24*60*60*7 || all_versions[name].include?(latestsrc)) + # Do not delete if the new version of the package hasn't been built for this arch yet + # but still delete it if it is no longer expected to be built. + next unless (used_here_srcs[latestsrc] || !arch_wanted("#{src_path}/#{latestsrc}", arch)) + end + old_binaries << filename + end + } + old_binaries +end + +if __FILE__ == $0 then + process +end diff --git a/modules/buildsystem/templates/cleaner_test.rb b/modules/buildsystem/templates/cleaner_test.rb new file mode 100644 index 00000000..804bd1b5 --- /dev/null +++ b/modules/buildsystem/templates/cleaner_test.rb @@ -0,0 +1,83 @@ +require 'cleaner' +require "test/unit" + +class TestCleaner < Test::Unit::TestCase + + @pkgs = [] + + def setpackages(pkgs) + @pkgs = pkgs + end + + def packages(path) + @pkgs.map{|p| + l2 = p.split(':') + sourcerpm = l2[0] + filename = l2[1] + buildtime = l2[2].to_i + yield(sourcerpm, filename, buildtime) + } + end + + def test_old + # Package was built on this arch and src.rpm for new version is 15d old + setpackages(['foo-43-1.src.rpm:libfoo2-43-1.armv7hl.rpm:43', 'foo-42-1.src.rpm:libfoo1-42-1.armv7hl.rpm:42']) + srcages = {} + srcages['foo'] = [ 'foo-43-1.src.rpm', Time.now.to_i - 15*24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal(['libfoo1-42-1.armv7hl.rpm'], check_binaries('armv7hl', srcs, srcages, '', '', nil)) + end + + def test_recent + # Package was built on this arch but src.rpm for new version is only 1d old + setpackages(['foo-43-1.src.rpm:foo-43-1.armv7hl.rpm:43', 'foo-42-1.src.rpm:foo-42-1.armv7hl.rpm:42']) + srcages = {} + srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal([], check_binaries('armv7hl', srcs, srcages, '', '', nil)) + end + + def test_arm_late + # Package was not yet built on this arch + setpackages(['foo-42-1.src.rpm:foo-42-1.armv7hl.rpm:42']) + srcages = {} + srcages['foo'] = [ 'foo-43.src.rpm', Time.now.to_i - 24*3600 ] + srcs = {} + srcs['foo-43-1.src.rpm'] = true + assert_equal([], check_binaries('armv7hl', srcs, srcages, '', '', nil)) + end + + def test_multiple_versions + # Old package remains (usually happens to noarch due to youri bug) + $noarch = { 'foo' => true } + setpackages(['foo-42-1.src.rpm:foo-42-1.noarch.rpm:42', 'foo-42-2.src.rpm:foo-42-2.noarch.rpm:43']) + srcages = {} + srcages['foo'] = [ 'foo-42-2.src.rpm', Time.now.to_i - 24*3600 ] + srcs = {} + srcs['foo-42-2.src.rpm'] = true + assert_equal(['foo-42-1.noarch.rpm'], check_binaries('i586', srcs, srcages, '', '', nil)) + end + + def test_icu + $noarch = {} + now = Time.now.to_i + srctime = now - 3600 + oldbintime = now - 10*24*3600 + newbintime = now - 3200 + setpackages([ + "icu-71.1-2.mga9.src.rpm:icu71-data-71.1-2.mga9.noarch.rpm:#{oldbintime}", + "icu-71.1-2.mga9.src.rpm:lib64icu71-71.1-2.mga9.aarch64.rpm:#{oldbintime}", + "icu-72.1-1.mga9.src.rpm:icu72-data-72.1-1.mga9.noarch.rpm:#{newbintime}", + "icu-72.1-1.mga9.src.rpm:lib64icu-devel-72.1-1.mga9.aarch64.rpm:#{newbintime}", + "icu-72.1-1.mga9.src.rpm:lib64icu72-72.1-1.mga9.aarch64.rpm:#{newbintime}" + ]) + srcages = {} + srcages['icu'] = [ 'icu-71.1-2.mga9.src.rpm', srctime ] + srcs = {} + srcs['icu-71.1-2.mga9.src.rpm'] = true + assert_equal([], check_binaries('aarch64', srcs, srcages, '', '', nil)) + end + +end diff --git a/modules/buildsystem/templates/iurt.cauldron.conf b/modules/buildsystem/templates/iurt.cauldron.conf deleted file mode 100644 index 5a7f047e..00000000 --- a/modules/buildsystem/templates/iurt.cauldron.conf +++ /dev/null @@ -1,30 +0,0 @@ -{ - supported_arch => [ 'i586', 'x86_64' ], - all_media => { 'main' => [ 'release' ], 'contrib' => [ 'release' ] }, - upload => 'schedbot@pkgsubmit:~/uploads/', - upload_queue => 'schedbot@pkgsubmit:~/uploads/queue/', - unwanted_packages => '^monotone-', - repository => '/mnt/BIG/dis/', - rsync_to => 'schedbot@pkgsubmit:/mnt/BIG/dis/uploads/build/', - log_url => 'http://pkgsubmit.mageia.org/queue/build/', - admin => 'mageia-sysadm@mageia.org', - iurt_root_command => '/home/buildbot/iurt-trunk/iurt_root_command', - packager => 'Iurt the rebuild bot <mageia-sysadm@mageia.org>', - sendmail => 0, - build_timeout => { - 'default' => 18000, - 'gcc' => 57600, - 'paraview' => 115200, - 'salome' => 57600, - 'itk' => 115200, - 'wrapitk' => 115200, - 'kernel-rt' => 57600, - 'kernel-xen' => 57600, - 'kernel-tmb' => 57600, - 'openoffice.org' => 345600, - 'openoffice.org64' => 345600, - 'openoffice.org-go-ooo' => 345600, - 'openoffice.org64-go-ooo' => 345600 - }, -} - diff --git a/modules/buildsystem/templates/iurt.conf b/modules/buildsystem/templates/iurt.conf new file mode 100644 index 00000000..2dd8bf0e --- /dev/null +++ b/modules/buildsystem/templates/iurt.conf @@ -0,0 +1,37 @@ +<%- distro = scope.lookupvar('buildsystem::var::distros::distros')[@distribution] -%> +{ + supported_arch => [ '<%= distro['arch'].join("', '") %>' ], + all_media =>{ +<%- distro['medias'].keys.sort.each{|media| -%> + '<%= media %>' => [ '<%= + distro['medias'][media]['repos'].keys.sort.join("', '") %>' ], +<%- +} -%> + }, + distribution => '<%= distro['macros']['distribution'] %>', + vendor => '<%= distro['macros']['vendor'] %>', + base_media => [ '<%= distro['base_media'].join("', '") %>' ], + upload => '<%= build_login %>@pkgsubmit:~/uploads/', + upload_queue => '<%= build_login %>@pkgsubmit:~/uploads/queue/', + unwanted_packages => '^monotone-', + repository => 'http://<%= scope.lookupvar('buildsystem::var::repository::hostname') %>/<%= scope.lookupvar('buildsystem::var::repository::distribdir') %>/', + log_url => 'https://<%= scope.lookupvar('buildsystem::var::webstatus::hostname') %>/queue/build/', + admin => 'mageia-sysadm@mageia.org', + packager => 'Iurt the rebuild bot <mageia-sysadm@mageia.org>', + sendmail => 0, + log_size_limit => '600M', + build_timeout => { +<%- build_timeout.keys.sort.each{|package| -%> + '<%= package %>' => <%= (build_timeout[package].to_f * scope.lookupvar('buildsystem::var::iurt::timeout_multiplier').to_f).to_i %>, +<%- +} -%> + }, + use_netns => { + 'default' => 1, +<%- allow_network_access.sort.each{|package| -%> + '<%= package %>' => 0, +<%- +} -%> + }, +} + diff --git a/modules/buildsystem/templates/maintdb/maintdb.bin b/modules/buildsystem/templates/maintdb/maintdb.bin new file mode 100755 index 00000000..903ee009 --- /dev/null +++ b/modules/buildsystem/templates/maintdb/maintdb.bin @@ -0,0 +1,98 @@ +#!/bin/bash + +MAINTDBDIR="<%= scope.lookupvar('buildsystem::var::maintdb::dbdir') %>" + +function checkname() +{ + if [ -z "$1" ] || + echo "$1" | grep -q '[/*{}%]' || + echo "$1" | fgrep -q '..' + then + echo "Error: invalid package name." >&2 + exit 1 + fi +} + +function maintnew() +{ + if [ "$user" != "root" ]; then + echo "Error: new is only allowed to root." >&2 + exit 1 + fi + checkname "$1" + maintfile="$MAINTDBDIR/$1" + if [ -f "$maintfile" ]; then + exit 0 + fi + echo "$2" > "$maintfile" +} + +function maintset() +{ + checkname "$1" + maintfile="$MAINTDBDIR/$1" + newmaint="$2" + if [ ! -f "$maintfile" ]; then + echo "Error: package $1 does not exist in maintdb." >&2 + exit 1 + fi + curmaint=$(cat "$maintfile") + if [ "$newmaint" = "nobody" ] || [[ "$newmaint" = *-team ]]; then + if [ "$curmaint" = "$user" ]; then + echo "$newmaint" > "$maintfile" + exit 0 + else + echo "Error: cannot set maintainer for $1." >&2 + exit 1 + fi + elif [ "$newmaint" = "$user" ]; then + if [ "$curmaint" = "nobody" ] || [[ "$curmaint" = *-team ]]; then + echo "$newmaint" > "$maintfile" + exit 0 + else + echo "Error: cannot set maintainer for $1." >&2 + exit 1 + fi + else + echo "Error: cannot set someone else as maintainer." >&2 + exit 1 + fi +} + +function maintgetall() +{ + cd "$MAINTDBDIR" + for file in *; do + echo "$file $(cat $file)" + done + exit 0 +} + +function maintget() +{ + if [ -z "$1" ]; then + maintgetall + fi + checkname "$1" + maintfile="$MAINTDBDIR/$1" + if [ -f "$maintfile" ]; then + cat "$maintfile" + else + echo "Error: package $1 does not exist in maintdb." >&2 + exit 1 + fi +} + +user="$1" +action="$2" + +if [ "$action" = "new" ]; then + maintnew "$3" "$4" +elif [ "$action" = "set" ]; then + maintset "$3" "$4" +elif [ "$action" = "get" ]; then + maintget "$3" +else + echo "Error: unknown command." >&2 + exit 2 +fi diff --git a/modules/buildsystem/templates/maintdb/sudoers.maintdb b/modules/buildsystem/templates/maintdb/sudoers.maintdb new file mode 100644 index 00000000..91c88e47 --- /dev/null +++ b/modules/buildsystem/templates/maintdb/sudoers.maintdb @@ -0,0 +1,4 @@ +%<%= scope.lookupvar('buildsystem::var::groups::packagers') %> ALL =(<%= scope.lookupvar('buildsystem::var::maintdb::login') %>) NOPASSWD: <%= scope.lookupvar('buildsystem::var::maintdb::binpath') %> [a-z]* get +%<%= scope.lookupvar('buildsystem::var::groups::packagers') %> ALL =(<%= scope.lookupvar('buildsystem::var::maintdb::login') %>) NOPASSWD: <%= scope.lookupvar('buildsystem::var::maintdb::binpath') %> [a-z]* [gs]et [a-zA-Z0-9]* +%<%= scope.lookupvar('buildsystem::var::groups::packagers') %> ALL =(<%= scope.lookupvar('buildsystem::var::maintdb::login') %>) NOPASSWD: <%= scope.lookupvar('buildsystem::var::maintdb::binpath') %> [a-z]* set [a-zA-Z0-9]* [a-z]* +<%= scope.lookupvar('buildsystem::var::scheduler::login') %> ALL =(<%= scope.lookupvar('buildsystem::var::maintdb::login') %>) NOPASSWD: <%= scope.lookupvar('buildsystem::var::maintdb::binpath') %> [a-z]* new [a-zA-Z0-9]* [a-z]* diff --git a/modules/buildsystem/templates/maintdb/vhost_maintdb.conf b/modules/buildsystem/templates/maintdb/vhost_maintdb.conf new file mode 100644 index 00000000..146413a7 --- /dev/null +++ b/modules/buildsystem/templates/maintdb/vhost_maintdb.conf @@ -0,0 +1,3 @@ +<Directory <%= scope.lookupvar('buildsystem::var::maintdb::dbdir') %>> + Options None +</Directory> diff --git a/modules/buildsystem/templates/maintdb/wrapper.maintdb b/modules/buildsystem/templates/maintdb/wrapper.maintdb new file mode 100644 index 00000000..fcf69dab --- /dev/null +++ b/modules/buildsystem/templates/maintdb/wrapper.maintdb @@ -0,0 +1,25 @@ +#!/bin/sh + +maintdbuser="<%= scope.lookupvar('buildsystem::var::maintdb::login') %>" +maintdbpath="<%= scope.lookupvar('buildsystem::var::maintdb::binpath') %>" +packagersgroup="<%= scope.lookupvar('buildsystem::var::groups::packagers') %>" + +function isingroup() +{ + grp="$1" + for group in `groups` + do if [ "$grp" = "$group" ] + then + return 0 + fi + done + return 1 +} + +if ! isingroup "$packagersgroup" +then + echo "You are not in $packagersgroup group." + exit 1 +fi + +sudo -u "$maintdbuser" "$maintdbpath" $(whoami) "$@" diff --git a/modules/buildsystem/templates/media.cfg b/modules/buildsystem/templates/media.cfg new file mode 100644 index 00000000..64757a2b --- /dev/null +++ b/modules/buildsystem/templates/media.cfg @@ -0,0 +1,142 @@ +<%- +def media_name(media, repo, type, archname) + name = [ media.capitalize ] + if archname != nil + name += [ archname ] + end + for r in repo.split('_') do + name += [ r.capitalize ] + end + if type != nil + name += [ type.capitalize ] + end + return name.join(' ') +end + +def media_out(name, media_hash) + media_out = "[%s]\n" % name + media_hash.keys.sort.each{|key| + value = media_hash[key] + if value != nil + media_out += "%s=%s\n" % [ key, value ] + end + } + return media_out +end +distro = scope.lookupvar('buildsystem::var::distros::distros')[@distro_name] +-%> +[media_info] +version=<%= distro['version'] %> +mediacfg_version=2 +branch=<%= distro['branch'] %> +<%- +if @arch != 'armv7hl' +-%> +arch=<%= @arch %> +<%- +end +-%> +xml-info=1 + +<%- +distro['medias'].keys.sort { |x,y| distro['medias'][x]['order'] <=> distro['medias'][y]['order'] }.each{|medianame| + media = distro['medias'][medianame] + media['repos'].keys.sort { |x,y| media['repos'][x]['order'] <=> media['repos'][y]['order'] }.each{|reponame| + repo = media['repos'][reponame] + media_type = [] + if media['media_type'] != nil + media_type += media['media_type'] + end + if repo['media_type'] != nil + media_type += repo['media_type'] + end + noauto=nil + if (media['noauto'] == '1') or (repo['noauto'] == '1') + noauto='1' + end + updates_for = nil + if repo['updates_for'] != nil + updates_for = [ medianame, repo['updates_for'] ].join('/') + end + -%><%= + media_out [ medianame, reponame ].join('/'), + :hdlist => [ 'hdlist', medianame, reponame ].join('_') + '.cz', + :name => media_name(medianame, reponame, nil, nil), + :srpms => [ '../../SRPMS', medianame, reponame ].join('/'), + :media_type => media_type.join(':'), + :updates_for => updates_for, + :noauto => noauto + + %> + <%-# debug -%> + <%- + debug_media_type = media_type + [ 'debug' ] + -%><%= + media_out [ 'debug', medianame, reponame ].join('/'), + :hdlist => [ 'hdlist_debug', medianame, reponame ].join('_') + '.cz', + :name => media_name(medianame, reponame, 'debug', nil), + :srpms => [ '../../SRPMS', medianame, reponame ].join('/'), + :media_type => debug_media_type.join(':'), + :noauto => '1' + + %> + <%-# source -%> + <%- + source_media_type = media_type + [ 'source' ] + -%><%= + media_out [ '../../SRPMS', medianame, reponame ].join('/'), + :hdlist => [ 'hdlist', medianame, reponame ].join('_') + '.src.cz', + :name => media_name(medianame, reponame, 'sources', nil), + :rpms => [ medianame, reponame ].join('/'), + :media_type => source_media_type.join(':'), + :noauto => '1' + + %> + <%-# we add 32bit media if arch is x86_64 -%> + <%- + if @arch == 'x86_64' and distro['arch'].include?('i586') + medianame32 = [ medianame, '32' ].join('') + -%><%= + media_out [ '../../i586/media', medianame, reponame ].join('/'), + :hdlist => [ 'hdlist', medianame32, reponame ].join('_') + '.src.cz', + :name => media_name(medianame, reponame, nil, '32bit'), + :media_type => media_type.join(':'), + :noauto => noauto + + %> + <%- + end + if @arch == 'x86_64' and distro['arch'].include?('i686') + medianame32 = [ medianame, '32' ].join('') + -%><%= + media_out [ '../../i686/media', medianame, reponame ].join('/'), + :hdlist => [ 'hdlist', medianame32, reponame ].join('_') + '.src.cz', + :name => media_name(medianame, reponame, nil, '32bit'), + :media_type => media_type.join(':'), + :noauto => noauto + + %> + <%- + end + } +} +if distro['based_on'] != nil + distro['based_on'].keys.sort.each{|bdistroname| + bdistro = distro['based_on'][bdistroname] + bdistro.keys.sort.each{|medianame| + media = bdistro[medianame] + for reponame in media + -%><%= + media_out [ bdistroname, medianame, reponame ].join('/'), + :hdlist => [ 'hdlist', bdistroname, medianame, + reponame ].join('_'), + :name => media_name([ medianame, bdistroname].join(''), reponame, nil, nil), + :media_type => 'base_distro', + :noauto => 1 + %> + <%- + end + } + } +end +-%> diff --git a/modules/buildsystem/templates/mga-youri-submit b/modules/buildsystem/templates/mga-youri-submit new file mode 100755 index 00000000..0d29d462 --- /dev/null +++ b/modules/buildsystem/templates/mga-youri-submit @@ -0,0 +1,2 @@ +#!/bin/sh +exec sudo /usr/local/bin/mga-youri-submit.wrapper "$@" diff --git a/modules/buildsystem/templates/mga-youri-submit.wrapper b/modules/buildsystem/templates/mga-youri-submit.wrapper new file mode 100755 index 00000000..66fc59bc --- /dev/null +++ b/modules/buildsystem/templates/mga-youri-submit.wrapper @@ -0,0 +1,36 @@ +#!/usr/bin/perl +# youri-submit wrapper + +use strict; +use warnings; +use Fcntl ':mode'; +use File::Basename; +use MDK::Common; + +my $log_dir = "$ENV{HOME}/submit-logs"; + +my $sudo_user = $ENV{SUDO_USER} or die "should be run through sudo"; +my @prog = ('perl', '-I/usr/share/mga-youri-submit/lib', '/usr/share/mga-youri-submit/bin/youri-submit'); + +my @options; +foreach my $arg (@ARGV) { + if ($arg =~ /^-?-(\S+)/) { + # drop prohibited options + if ($arg =~ /-c/ || $arg =~ /-s/) { + print STDERR "prohibited option $arg, skipping\n"; + next; + } + } + push(@options, $arg); +} + +# logging for bug #30315 -spuk, 2007-05-29 +mkdir_p($log_dir); +open(STDERR, "| tee -a $log_dir/$sudo_user.err >&2"); +open(STDOUT, "| tee -a $log_dir/$sudo_user.out"); + +# call wrapped program +print "Executing @prog --config /etc/youri/submit-todo.conf --define user=$sudo_user @options (sudo_user $sudo_user)\n"; +my $err = system(@prog, "--config", "/etc/youri/submit-todo.conf", "--define", "user=$sudo_user", @options) && ($? >> 8 || 1); + +exit $err diff --git a/modules/buildsystem/templates/mgarepo.conf b/modules/buildsystem/templates/mgarepo.conf new file mode 100644 index 00000000..fbe5109c --- /dev/null +++ b/modules/buildsystem/templates/mgarepo.conf @@ -0,0 +1,88 @@ +<%- + default_distro = scope.lookupvar('buildsystem::var::distros::default_distro') + distros = scope.lookupvar('buildsystem::var::distros::distros') +-%> +[global] +verbose = no +default_parent = <%= scope.lookupvar('buildsystem::var::mgarepo::svn_root_packages') %>/<%= default_distro %> +#url-map = svn\+ssh://svn\.mageia\.org/(.*) file:///\1 +tempdir = <%= sched_home_dir %>/repsys/tmp +trunk-dir = <%= default_distro %> +<%- +conf = scope.lookupvar('buildsystem::var::mgarepo::conf') +if conf['global'] != nil + conf['global'].keys.sort.each{|key| + value = conf['global'][key] + -%><%= key %> = <%= value %> +<%- + } +end +-%> + + +[log] +oldurl = <%= scope.lookupvar('buildsystem::var::mgarepo::oldurl') %> +# controls up to which revision the rpm changelog +# will be constructed (default zero, i.e., oldest +# commit) +# revision-offset = 0 +# commit lines containing this string won't be shown in the changelog: +ignore-string = SILENT + +[template] +path = /usr/share/mgarepo/default.chlog + +[users] +iurt = Mageia build bot <mageia-sysadm@<%= domain %>> + +[submit] +default = <%= default_distro %> +host = <%= scope.lookupvar('buildsystem::var::mgarepo::submit_host') %> + +<%- + distros.keys.sort.each{|d| + distro = distros[d] +-%> +[submit <%= d %>] +target = <%= sched_home_dir %>/repsys/srpms +allowed = <%= distro['submit_allowed'] %> <%= distro['backports_allowed'] %> +rpm-macros = global <%= d %> + +<%- + } +-%> + +[macros global] +# mkrel definition to be removed when rpm-setup is updated on main build node +mkrel(c:) = %{-c: 0.%{-c*}.}%{1}%{?subrel:.%subrel}%{?distsuffix:%distsuffix}%{?!distsuffix:.mga}%{?distro_release:%distro_release} +dist = %{?distsuffix:%distsuffix}%{?!distsuffix:.mga}%{?distro_release:%distro_release} + +<%- + distros.keys.sort.each{|d| + distro = distros[d] +-%> +[macros <%= d %>] +distro_release = <%= distro['version'] %> +<%- + distro['macros'].keys.sort.each{|macro| + value = distro['macros'][macro] + -%><%= macro %> = <%= value %> + <%- } %> +<%- } +%> + +[helper] +create-srpm = /usr/share/repsys/create-srpm +upload-srpm = /usr/local/bin/mga-youri-submit +# needed by mdvsys 2.0 +install-buildrequires = sudo rurpmi --auto --no-suggests +upload-bin = /usr/local/bin/wrapper.upload-bin + +[srpm] +run-prep = yes + +[binrepo] +<%- binrepo_hostname = scope.lookupvar('buildsystem::var::binrepo::hostname') -%> +download_url = http://<%= binrepo_hostname %>/ +upload_host = <%= binrepo_hostname %> + diff --git a/modules/buildsystem/templates/repoctl.conf b/modules/buildsystem/templates/repoctl.conf new file mode 100644 index 00000000..14506a25 --- /dev/null +++ b/modules/buildsystem/templates/repoctl.conf @@ -0,0 +1,40 @@ +<%- +distribdir = scope.lookupvar('buildsystem::var::repository::distribdir') +distros = scope.lookupvar('buildsystem::var::distros::distros') +arches = {} +distrosections = {} +sectionsrepos = {} +distros.each{|distroname, distro| + distro['medias'].each{|medianame, media| + distrosections[medianame] = 1 + media['repos'].each{|reponame, repo| + sectionsrepos[reponame] = 1 + } + } + distro['arch'].each{|arch| + arches[arch] = 1 + } +} +-%> +dryrun=echo +if [ -z $SUDO_USER ] +then + requestuser="$USER" +else + requestuser="$SUDO_USER" +fi +lockdir=/var/lib/repoctl/locks +hdlistsdir=/var/lib/repoctl/hdlists +rootdir=<%= scope.lookupvar('buildsystem::var::repository::bootstrap_root') %> +finalrootdir=<%= scope.lookupvar('buildsystem::var::repository::mirror_root') %> +distribdir=$rootdir/<%= distribdir %> +finaldistribdir=$finalrootdir/<%= distribdir %> +distroreleases='<%= distros.keys.sort.join(' ') -%>' +distrosections='<%= distrosections.keys.sort.join(' ') -%>' +sectionsrepos='<%= sectionsrepos.keys.sort.join(' ') -%>' +arches='<%= arches.keys.sort.join(' ') -%>' +mirror_rsync_options="-v --delete -alH" +timestampfile="mageia_timestamp" +sha1sumfile="mageia_sha1sum" +sha1sumsigfile="mageia_sha1sum.gpg" +sign_mirror_sha1sum=/bin/true diff --git a/modules/buildsystem/templates/rpmlint.conf b/modules/buildsystem/templates/rpmlint.conf new file mode 100644 index 00000000..b81f169b --- /dev/null +++ b/modules/buildsystem/templates/rpmlint.conf @@ -0,0 +1,7 @@ +from Config import * +execfile('/etc/rpmlint/extracted.d/distribution.exceptions.conf') + +for i in open('/etc/rpmlint/extracted.d/distribution.error.list').readlines(): + setBadness(i, 10) + + diff --git a/modules/buildsystem/templates/signbot/sudoers.signpackage b/modules/buildsystem/templates/signbot/sudoers.signpackage new file mode 100644 index 00000000..4ea30238 --- /dev/null +++ b/modules/buildsystem/templates/signbot/sudoers.signpackage @@ -0,0 +1,2 @@ +<%= sched_login %> ALL =(<%= scope.lookupvar('buildsystem::var::signbot::login') %>) NOPASSWD: /usr/local/bin/mga-signpackage +<%= sched_login %> ALL =(<%= scope.lookupvar('buildsystem::var::signbot::login') %>) NOPASSWD: /usr/local/bin/sign-check-package diff --git a/modules/buildsystem/templates/submit_package.pl b/modules/buildsystem/templates/submit_package.pl new file mode 100755 index 00000000..1fdf7749 --- /dev/null +++ b/modules/buildsystem/templates/submit_package.pl @@ -0,0 +1,18 @@ +#!/usr/bin/perl +use strict; +use warnings; + +my $svn_server = '<%= scope.lookupvar('buildsystem::var::mgarepo::svn_hostname') %>'; +my $packagersgroup="<%= scope.lookupvar('buildsystem::var::groups::packagers') %>"; + +my $login = getpwuid($<); +my (undef, undef, undef, $members) = getgrnam $packagersgroup; +if (not $members =~ /\b$login\b/) { + print "You are not in $packagersgroup group\n"; + exit 1; +} + +# for bug 914 +# https://bugs.mageia.org/show_bug.cgi?id=914 +map { $_ =~ s|^svn\+ssh://$svn_server/|svn://$svn_server/| } @ARGV; +exec "/usr/share/mgarepo/create-srpm", @ARGV; diff --git a/modules/buildsystem/templates/sudoers.iurt b/modules/buildsystem/templates/sudoers.iurt index 266f301c..21e81e87 100644 --- a/modules/buildsystem/templates/sudoers.iurt +++ b/modules/buildsystem/templates/sudoers.iurt @@ -1 +1 @@ -<%= build_login %> ALL = NOPASSWD: /usr/sbin/iurt_root_command +<%= scope.lookupvar('buildsystem::var::iurt::login') %> ALL = NOPASSWD: /usr/sbin/iurt_root_command diff --git a/modules/buildsystem/templates/sudoers.youri b/modules/buildsystem/templates/sudoers.youri new file mode 100644 index 00000000..3bc7cc2d --- /dev/null +++ b/modules/buildsystem/templates/sudoers.youri @@ -0,0 +1,6 @@ +<%- sched_login = scope.lookupvar('buildsystem::var::scheduler::login') -%> +Cmnd_Alias YOURI = /usr/local/bin/mga-youri-submit.wrapper +Defaults!YOURI always_set_home +Defaults!YOURI runas_default = <%= sched_login %> +Defaults!YOURI !requiretty +%<%= scope.lookupvar('buildsystem::var::groups::packagers') -%> ALL = (<%= sched_login %>) NOPASSWD: YOURI diff --git a/modules/buildsystem/templates/upload.conf b/modules/buildsystem/templates/upload.conf new file mode 100644 index 00000000..af610c92 --- /dev/null +++ b/modules/buildsystem/templates/upload.conf @@ -0,0 +1,131 @@ +### +# +# Do not disable the host without appropriate warning +# to somebody able to fix the machine +# +# Please run 'perl -cw .upload.conf' in order to check the file is OK. +# +### + +<%- + build_nodes = scope.lookupvar('buildsystem::var::scheduler::build_nodes') +-%> +my %nodes = ( +<%- + build_nodes.keys.sort.each{|arch| +-%> + <%= arch -%> => [ '<%= build_nodes[arch].join("', '") -%>' ], +<%- + } +-%> +); +my $repository = "http://<%= scope.lookupvar('buildsystem::var::repository::hostname') %>/<%= scope.lookupvar('buildsystem::var::repository::distribdir') %>/"; +my $homedir = "<%= scope.lookupvar('buildsystem::var::iurt::homedir') %>"; + +{ + bot => { + (map { + my $arch = $_; + $arch => { + map { + my $node = $_; + ($node => { + iurt => { + user => '<%= scope.lookupvar('buildsystem::var::iurt::login') %>', +# (spuk, 2007-08-16) disabled iurt_cache additional media, locks trying to mount -o bind +# command => "iurt --copy-srpm --group -v 6 --config local_spool $homedir/iurt/__DIR__ --no_rsync --chrooted-urpmi -m __MEDIA__ -- $repository --additional-media -m __MEDIA__ -- file://$homedir/cache/ -p \"__PACKAGER__\" -r __TARGET__ __ARCH__", + command => "iurt --copy_srpm --group --rpmmacros \"%distro_section __SECTION__\" --config local_spool $homedir/iurt/__DIR__ --no_rsync --chrooted-urpmi -m __MEDIA__ -- $repository -p \"__PACKAGER__\" -r __TARGET__ __ARCH__", + packages => "$homedir/iurt/", + }, + }); + } @{$nodes{$arch}}, + }; + } keys %nodes), + }, + media => { + <%- + def repo_deps(distros, dname, mname, rname) + deps = {} + distro = distros[dname] + if distro['based_on'] != nil + distro['based_on'].each{|bdistro, bmedias| + if bmedias[mname] != nil and \ + bmedias[mname].include?(rname) then + deps[ [ bdistro, mname, rname ].join('/') ] = 1 + end + } + end + if distro['medias'][mname] != nil \ + and distro['medias'][mname]['repos'][rname] != nil + then + deps[ [ mname, rname ].join('/') ] = 1 + else + return deps + end + mlist = distro['medias'][mname]['requires'] + mlist = mlist == nil ? [ mname ] : [ mname ] + mlist + mlist.each{|mreq| + rlist = distro['medias'][mname]['repos'][rname]['requires'] + rlist = [] if rlist == nil + rlist += [ rname ] if mreq != mname + rlist.each{|rreq| + deps.merge!(repo_deps(distros, dname, mreq, rreq)) + } + } + return deps + end + distros = scope.lookupvar('buildsystem::var::distros::distros') + distros.keys.sort.each{|distroname| + -%> + '<%= distroname -%>' => { + <%- + distro = distros[distroname] + distro['medias'].keys.sort.each{|medianame| + media = distro['medias'][medianame] + media['repos'].keys.sort.each{|reponame| + deps = repo_deps(distros, distroname, medianame, reponame) + -%> + "<%= [ medianame, reponame ].join('/') %>" => [ "<%= + deps.keys.sort.join('", "') + %>" ], + <%- + } + } + -%> + }, + <%- + } + -%> + }, + admin => '<%= scope.lookupvar('buildsystem::var::scheduler::admin_mail') %>', + http_queue => 'https://<%= scope.lookupvar('buildsystem::var::webstatus::hostname') %>/uploads', + upload_user => '<%= scope.lookupvar('buildsystem::var::scheduler::login') %>', + email_domain => '<%= domain %>', + arch => { + <%- + distros.keys.sort.each{|distroname| + -%> + <%= distroname -%> => [ '<%= distros[distroname]['arch'].join("', '") %>' ], + <%- + } + -%> + default => [ 'i586', 'x86_64' ], + }, + mandatory_arch => { + <%- + distros.keys.sort.each{|distroname| + if distros[distroname]['mandatory_arch'] != nil + march = distros[distroname]['mandatory_arch'] + else + march = distros[distroname]['arch'] + end + -%> + <%= distroname -%> => [ '<%= march.join("', '") %>' ], + <%- + } + -%> + default => [ 'i586', 'x86_64' ], + }, + ssh_options => "-o ServerAliveInterval=10 -o ConnectTimeout=20 -o BatchMode=yes", + faildelay => 360000, +} diff --git a/modules/buildsystem/templates/vhost_repository.conf b/modules/buildsystem/templates/vhost_repository.conf new file mode 100644 index 00000000..e082ffca --- /dev/null +++ b/modules/buildsystem/templates/vhost_repository.conf @@ -0,0 +1,73 @@ +<%- +mirror_root = scope.lookupvar('buildsystem::var::repository::mirror_root') +mirror_reporoot = scope.lookupvar('buildsystem::var::repository::mirror_reporoot') +bootstrap_reporoot = scope.lookupvar('buildsystem::var::repository::bootstrap_reporoot') +distribdir = scope.lookupvar('buildsystem::var::repository::distribdir') +repo_allow_from_ips = scope.lookupvar('buildsystem::var::distros::repo_allow_from_ips') +repo_allow_from_domains = scope.lookupvar('buildsystem::var::distros::repo_allow_from_ips') +distros = scope.lookupvar('buildsystem::var::distros::distros') +-%> +<VirtualHost *:80> + ServerName <%= scope.lookupvar('buildsystem::var::repository::hostname') %> + DocumentRoot <%= mirror_root %> + + # Some simple API to check existence of SRPMs for QA + RewriteEngine On + + RewriteCond /distrib/bootstrap/distrib/$2/SRPMS/$3/$1s_testing/$4.src.rpm -f + RewriteRule ^/qa/checksrpm/(update|backport)/([1-9][0-9]*)/([a-z_]+)/([^/]+)$ http://repository.mageia.org/qa/checksrpm/found [L,R=302] + + RewriteRule ^/qa/checksrpm/ - [L,G] + +<%- + if repo_allow_from_ips != nil || repo_allow_from_domains != nil then + access_requires = [ 'all denied' ] + if repo_allow_from_ips != nil then + for allow in repo_allow_from_ips do + access_requires << 'ip ' + allow + end + end + if repo_allow_from_domains != nil then + for allow in repo_allow_from_domains do + access_requires << 'host ' + allow + end + end + else + access_requires = [ 'all granted' ] + end +%> + Alias /bootstrap/ "<%= bootstrap_reporoot %>/" +<%- + distros.keys.sort.each{|distroname| + distro = distros[distroname] + if distro['no_mirror'] -%> + Alias /<%= distribdir %>/<%= distroname %>/ "<%= bootstrap_reporoot %>/<%= distroname %>/" +<%- + end + } +-%> + + <Directory <%= bootstrap_reporoot %>> + Header append Cache-Control "public, must-revalidate" +<%- + for req in access_requires do +-%> + Require <%= req %> +<%- + end +-%> + Options Indexes FollowSymLinks + </Directory> + + <Directory <%= mirror_root %>> + Header append Cache-Control "public, must-revalidate" +<%- + for req in access_requires do +-%> + Require <%= req %> +<%- + end +-%> + Options Indexes FollowSymLinks + </Directory> +</VirtualHost> diff --git a/modules/buildsystem/templates/vhost_webstatus.conf b/modules/buildsystem/templates/vhost_webstatus.conf new file mode 100644 index 00000000..3b0e6246 --- /dev/null +++ b/modules/buildsystem/templates/vhost_webstatus.conf @@ -0,0 +1,13 @@ +<Location /uploads> + Require all granted + Options Indexes + IndexOptions NameWidth=* +</Location> +<Location /autobuild> + Require all granted + Options Indexes + IndexOptions NameWidth=* +</Location> +<Directory /var/www/bs/autobuild> + Options FollowSymlinks FollowSymLinks +</Directory> diff --git a/modules/buildsystem/templates/youri/acl.conf b/modules/buildsystem/templates/youri/acl.conf new file mode 100644 index 00000000..f0949f8a --- /dev/null +++ b/modules/buildsystem/templates/youri/acl.conf @@ -0,0 +1 @@ +.* .* .* ^glib$ ^blacklisted$ diff --git a/modules/buildsystem/templates/youri/host.conf b/modules/buildsystem/templates/youri/host.conf new file mode 100644 index 00000000..bf4fa086 --- /dev/null +++ b/modules/buildsystem/templates/youri/host.conf @@ -0,0 +1,23 @@ +<%- + aliases = scope.lookupvar('buildsystem::var::scheduler::build_nodes_aliases') + nodes = {} + nodes['src'] = [ scope.lookupvar('buildsystem::var::scheduler::build_src_node') ] + scope.lookupvar('buildsystem::var::scheduler::build_nodes').each{|arch,n| + a = arch + '|noarch|src' + nodes[a] = [] + n.each{|node| + if aliases[node] != nil + nodes[a] += [ aliases[node] ] + else + nodes[a] += [ node ] + end + } + } + str = '' + nodes.keys.sort.each{|arch| + nodes[arch].sort.uniq.each{|node| + str += node + ' ' + arch + "\n" + } + } +-%> +<%= str -%> diff --git a/modules/buildsystem/templates/youri/submit.conf b/modules/buildsystem/templates/youri/submit.conf new file mode 100644 index 00000000..0d7cf927 --- /dev/null +++ b/modules/buildsystem/templates/youri/submit.conf @@ -0,0 +1,140 @@ +<% +Puppet::Parser::Functions.autoloader.loadall +sched_home_dir = scope.lookupvar('buildsystem::var::scheduler::homedir') + +conf = scope.lookupvar('buildsystem::var::youri::youri_conf') +conf_default = scope.lookupvar('buildsystem::var::youri::youri_conf_default') +distros = scope.lookupvar('buildsystem::var::distros::distros') + +def line(text, indent) + res = '' + i = 0 + while i < indent + res += ' ' + i += 1 + end + res += text + "\n" +end + +def array_text(array, indent) + res = '' + array.each{|a| + res += line('- ' + a, indent) + } + return res +end + +def hash_text(hash, indent) + res = '' + curindent = indent + hash.keys.sort.each{|key| + if hash[key].instance_of? Hash + res += line(key + ':', indent) + res += hash_text(hash[key], indent + 4) + elsif hash[key].instance_of? Array + res += line(key + ':', indent) + res += array_text(hash[key], indent + 4) + elsif hash[key].instance_of? String + res += line(key + ': ' + hash[key], indent) + end + } + return res +end + +def class_hash(conf, conf_default) + res = {} + res['class'] = get_conf(conf, ['class']) == nil ? + conf_default['class'] : conf['class'] + res['options'] = get_conf(conf_default, ['options']) == nil ? {} : + conf_default['options'].dup + if get_conf(conf, ['options']) != nil + res['options'].merge!(conf['options']) + end + return res +end + +def get_conf(conf, path) + res = conf + path.each{|p| + if res == nil + return nil + end + res = res[p] + } + return res +end + +def get_distros_conf(distros, conf_name, path) + res = {} + distros.keys.each{|distro| + t = get_conf(distros[distro], [ 'youri', conf_name ] + path) + if t != nil + res[distro] = t.dup + end + } + return res +end + +def get_definitions(def_name, conf_name, conf, conf_default, distros) + res = {} + res[def_name] = {} + def_list = conf_default[conf_name][def_name].keys + def_list += get_conf(conf, [ conf_name, def_name ]) != nil ? \ + conf[conf_name][def_name].keys : [] + def_list.uniq.each{|d| + res[def_name][d] = class_hash( + get_conf(conf, [ conf_name, def_name, d ]), + get_conf(conf_default, [ conf_name, def_name, d ]) + ) + res[def_name][d]['options'].merge!(get_distros_conf(distros, + conf_name, [ def_name, d ])) + } + return res +end + +%> +home: <%= sched_home_dir %> + +<%- + repository = { + 'repository' => class_hash(get_conf(conf[conf_name], + ['repository']), + conf_default[conf_name]['repository']), + } + distros.keys.each{|distro| + repository['repository']['options'][distro] = { + 'arch' => distros[distro]['arch'].join(' '), + } + } +-%> +# repository declaration +<%= hash_text(repository, 0) %> + +<%- + targets = { + 'targets' => get_distros_conf(distros, conf_name, [ 'targets' ]), + } +-%> +# targets definition +<%= hash_text(targets, 0) %> + +<%- + checks = get_definitions('checks', conf_name, conf, conf_default, distros) +-%> +# checks definition +<%= hash_text(checks, 0) -%> + +<%- + actions = get_definitions('actions', conf_name, conf, conf_default, distros) +-%> +# actions definitions +<%= hash_text(actions, 0) -%> + +<%- + posts = get_definitions('posts', conf_name, conf, conf_default, distros) +-%> + +# posts definitions +<%= hash_text(posts, 0) -%> + +# vim:ft=yaml:et:sw=4 |