diff options
Diffstat (limited to 'modules/buildsystem/manifests')
39 files changed, 1101 insertions, 306 deletions
diff --git a/modules/buildsystem/manifests/base.pp b/modules/buildsystem/manifests/base.pp deleted file mode 100644 index 8befa33d..00000000 --- a/modules/buildsystem/manifests/base.pp +++ /dev/null @@ -1,4 +0,0 @@ -class buildsystem::base { - $packagers_group = 'mga-packagers' - $packagers_committers_group = 'mga-packagers-committers' -} diff --git a/modules/buildsystem/manifests/binrepo.pp b/modules/buildsystem/manifests/binrepo.pp index 037352a0..5bf16b53 100644 --- a/modules/buildsystem/manifests/binrepo.pp +++ b/modules/buildsystem/manifests/binrepo.pp @@ -1,29 +1,29 @@ class buildsystem::binrepo { - include buildsystem::base + include buildsystem::var::binrepo + include buildsystem::var::groups include sudo - $login = 'binrepo' - $homedir = "/var/lib/$login" - $repodir = "$homedir/data" - $uploadinfosdir = "$homedir/infos" - $uploadbinpath = '/usr/local/bin/upload-bin' - $uploadmail_from = "root@$::domain" - $uploadmail_to = "packages-commits@ml.$::domain" - - # used in templates - $packagers_committers_group = $buildsystem::base::packagers_committers_group + # upload-bin script uses the mailx command provided by nail +if versioncmp($::lsbdistrelease, '9') < 0 { + package { 'nail': + ensure => installed, + } +} else { + package { 's-nail': + ensure => installed, + } +} - user { $login: - comment => 'Binary files repository', - home => $homedir, + user { $buildsystem::var::binrepo::login: + home => $buildsystem::var::binrepo::homedir, } - file { [$repodir, $uploadinfosdir]: + file { [$buildsystem::var::binrepo::repodir, $buildsystem::var::binrepo::uploadinfosdir]: ensure => directory, - owner => $login, + owner => $buildsystem::var::binrepo::login, } - mga-common::local_script { + mga_common::local_script { 'upload-bin': content => template('buildsystem/binrepo/upload-bin'); 'wrapper.upload-bin': @@ -34,8 +34,15 @@ class buildsystem::binrepo { content => template('buildsystem/binrepo/sudoers.binrepo') } - apache::vhost::base { "binrepo.$::domain": - location => $repodir, + apache::vhost::base { $buildsystem::var::binrepo::hostname: + location => $buildsystem::var::binrepo::repodir, + content => template('buildsystem/binrepo/vhost_binrepo.conf'), + } + + apache::vhost::base { "ssl_${buildsystem::var::binrepo::hostname}": + use_ssl => true, + vhost => $buildsystem::var::binrepo::hostname, + location => $buildsystem::var::binrepo::repodir, content => template('buildsystem/binrepo/vhost_binrepo.conf'), } } diff --git a/modules/buildsystem/manifests/buildnode.pp b/modules/buildsystem/manifests/buildnode.pp index bf898f32..1573c093 100644 --- a/modules/buildsystem/manifests/buildnode.pp +++ b/modules/buildsystem/manifests/buildnode.pp @@ -1,11 +1,12 @@ class buildsystem::buildnode { - include buildsystem::base include buildsystem::iurt - include buildsystem::scheduler::var + include buildsystem::var::scheduler + include buildsystem::var::iurt include buildsystem::sshkeys - # permit to scheduler to run iurt - ssh::auth::server { $buildsystem::scheduler::var::login: - user => $buildsystem::iurt::user::login, + sshkeys::set_authorized_keys { 'iurt-allow-scheduler': + keyname => $buildsystem::var::scheduler::login, + home => $buildsystem::var::iurt::homedir, + user => $buildsystem::var::iurt::login, } } diff --git a/modules/buildsystem/manifests/config.pp b/modules/buildsystem/manifests/config.pp deleted file mode 100644 index b038da21..00000000 --- a/modules/buildsystem/manifests/config.pp +++ /dev/null @@ -1,10 +0,0 @@ -class buildsystem::config( - architectures = ['i586', 'x86_64'], - dev_distros = ['cauldron'], - stable_distros = ['1', '2'], - distrosections = ['core', 'nonfree', 'tainted'], - sectionsrepos = ['release', 'updates', 'updates_testing', 'backports', - 'backports_testing'] -) -{ -} diff --git a/modules/buildsystem/manifests/create_upload_dir.rb b/modules/buildsystem/manifests/create_upload_dir.rb index dd30d00d..8023ab5d 100644 --- a/modules/buildsystem/manifests/create_upload_dir.rb +++ b/modules/buildsystem/manifests/create_upload_dir.rb @@ -1,22 +1,25 @@ -define "create_upload_dir", :owner, :group, :releases do +hostclass "buildsystem::create_upload_dir" do states = ["todo","done","failure","queue","rejected"] + owner = scope.lookupvar('buildsystem::var::scheduler::login') + group = owner + uploads_dir = scope.lookupvar('buildsystem::var::scheduler::homedir') + '/uploads' - file @name, :ensure => 'directory', :owner => @owner, :group => @group + file uploads_dir, :ensure => 'directory', :owner => owner, :group => group for st in states do - file [@name, st].join('/'), :ensure => 'directory', :owner => @owner, :group => @group - - @releases.each{|rel, repositories| - file [@name, st, rel].join('/'), :ensure => 'directory', :owner => @owner, :group => @group - - repositories.each{|rep, medias| - file [@name, st, rel, rep].join('/'), :ensure => 'directory', :owner => @owner, :group => @group - - for med in medias do + file [uploads_dir, st].join('/'), :ensure => 'directory', :owner => owner, :group => group + + scope.lookupvar('buildsystem::var::distros::distros').each{|rel, distro| + file [uploads_dir, st, rel].join('/'), :ensure => 'directory', :owner => owner, :group => group + medias = distro['medias'] + medias.each{|media, m| + file [uploads_dir, st, rel, media].join('/'), :ensure => 'directory', :owner => owner, :group => group + + for repo in m['repos'].keys do if st == 'done' - file [@name, st, rel, rep, med].join('/'), :ensure => 'directory', :owner => @owner, :group => @group, :mode => 0775 + file [uploads_dir, st, rel, media, repo].join('/'), :ensure => 'directory', :owner => owner, :group => group, :mode => 0775 else - file [@name, st, rel, rep, med].join('/'), :ensure => 'directory', :owner => @owner, :group => @group + file [uploads_dir, st, rel, media, repo].join('/'), :ensure => 'directory', :owner => owner, :group => group end end } diff --git a/modules/buildsystem/manifests/distros.rb b/modules/buildsystem/manifests/distros.rb new file mode 100644 index 00000000..a298c0a8 --- /dev/null +++ b/modules/buildsystem/manifests/distros.rb @@ -0,0 +1,97 @@ +hostclass "buildsystem::distros" do + mirror_user = 'root' + schedbot_user = scope.lookupvar('buildsystem::var::scheduler::login') + bootstrap_reporoot = scope.lookupvar('buildsystem::var::repository::bootstrap_reporoot') + scope.lookupvar('buildsystem::var::distros::distros').each{|rel, distro| + file [ bootstrap_reporoot, rel ].join('/'), :ensure => 'directory', + :owner => mirror_user, :group => mirror_user + for arch in distro['arch'] do + # As ruby dsl cannot use defined resources, we have to use a + # workaround with 'find_resource_type' as described in this + # puppet issue: http://projects.puppetlabs.com/issues/11912 + scope.find_resource_type 'buildsystem::media_cfg' + media_cfg_args = { + :distro_name => rel, + :arch => arch, + } + if distro['tmpl_media.cfg'] != nil + media_cfg_args['templatefile'] = distro['tmpl_media.cfg'] + end + if ! distro['no_media_cfg_update'] + create_resource 'buildsystem::media_cfg', + [ rel, ' ', arch ].join('/'), media_cfg_args + end + file [ bootstrap_reporoot, rel, arch ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + mediadir = [ bootstrap_reporoot, rel, arch, 'media' ].join('/') + file mediadir, :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, 'media_info' ].join('/'), :ensure => 'directory', + :owner => schedbot_user, :group => schedbot_user + file [ mediadir, 'debug' ].join('/'), :ensure => 'directory', + :owner => schedbot_user, :group => schedbot_user + distro['medias'].each{|media, m| + file [ mediadir, media ].join('/'), :ensure => 'directory', + :owner => schedbot_user, :group => schedbot_user + file [ mediadir, 'debug', media ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + for repo in m['repos'].keys do + file [ mediadir, media, repo ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, media, repo, 'media_info' ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, media, repo, 'repodata' ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, 'debug', media, repo ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, 'debug', media, repo, 'media_info' ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + file [ mediadir, 'debug', media, repo, 'repodata' ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + end + } + if distro['based_on'] != nil + distro['based_on'].each{|bdistroname, medias| + file [ mediadir, bdistroname ].join('/'), + :ensure => 'directory', :owner => mirror_user, + :group => mirror_user + medias.each{|medianame, media| + mdir = [ mediadir, bdistroname, medianame ].join('/') + file mdir, :ensure => 'directory', + :owner => mirror_user, :group => mirror_user + for reponame in media + file [ mdir, reponame ].join('/'), + :ensure => 'link', + :target => [ + '../../../../..', bdistroname, arch, + 'media', medianame, reponame ].join('/'), + :owner => mirror_user, :group => mirror_user + end + } + } + end + end + # SRPMS + srpmsdir = [ bootstrap_reporoot, rel, 'SRPMS' ].join('/') + file srpmsdir, + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + distro['medias'].each{|media, m| + file [ srpmsdir, media ].join('/'), :ensure => 'directory', + :owner => schedbot_user, :group => schedbot_user + for repo in m['repos'].keys do + file [ srpmsdir, media, repo ].join('/'), + :ensure => 'directory', :owner => schedbot_user, + :group => schedbot_user + end + } + } +end diff --git a/modules/buildsystem/manifests/init.pp b/modules/buildsystem/manifests/init.pp index 65b38b62..f15b5dbf 100644 --- a/modules/buildsystem/manifests/init.pp +++ b/modules/buildsystem/manifests/init.pp @@ -1,18 +1,2 @@ class buildsystem { - # A script to copy on valstar the 2010.1 rpms built on jonund - class sync20101 inherits base { - $build_login = $buildsystem::iurt::user::login - # TODO user iurt::user::homedir too - mga-common::local_script { 'sync2010.1': - content => template('buildsystem/sync2010.1'), - } - } - - # a script to build 2010.1 packages. used on jonund - class iurt20101 inherits base { - $build_login = $buildsystem::iurt::user::login - mga-common::local_script { 'iurt2010.1': - content => template('buildsystem/iurt2010.1'), - } - } } diff --git a/modules/buildsystem/manifests/iurt.pp b/modules/buildsystem/manifests/iurt.pp index e0856a23..231c5373 100644 --- a/modules/buildsystem/manifests/iurt.pp +++ b/modules/buildsystem/manifests/iurt.pp @@ -2,13 +2,11 @@ class buildsystem::iurt { include sudo include buildsystem::iurt::user include buildsystem::iurt::packages - $login = $buildsystem::iurt::user::login - $homedir = $buildsystem::iurt::user::homedir - - ssh::auth::client { $login: } + include buildsystem::var::iurt + include buildsystem::var::distros # remove old build directory - tidy { "$homedir/iurt": + tidy { "${buildsystem::var::iurt::homedir}/iurt": age => '8w', recurse => true, matches => ['[0-9][0-9].*\..*\..*\.[0-9]*','log','*.rpm','*.log','*.mga[0-9]+'], @@ -19,9 +17,10 @@ class buildsystem::iurt { ensure => directory, } - buildsystem::iurt::config { ['2','1','cauldron','mandriva2010.1','infra_1', 'infra_2']: } + $distros_list = hash_keys($buildsystem::var::distros::distros) + buildsystem::iurt::config { $distros_list: } sudo::sudoers_config { 'iurt': - content => template('buildsystem/iurt/sudoers.iurt') + content => template('buildsystem/sudoers.iurt') } } diff --git a/modules/buildsystem/manifests/iurt/config.pp b/modules/buildsystem/manifests/iurt/config.pp index 88c08b66..02f5be63 100644 --- a/modules/buildsystem/manifests/iurt/config.pp +++ b/modules/buildsystem/manifests/iurt/config.pp @@ -1,11 +1,50 @@ define buildsystem::iurt::config() { + include buildsystem::var::iurt + include buildsystem::var::webstatus + include buildsystem::var::repository $distribution = $name # TODO rename the variable too in template - $build_login = $buildsystem::iurt::user::login + $build_login = $buildsystem::var::iurt::login - file { "/etc/iurt/build/$distribution.conf": + $build_timeout = { + 'default' => 36000, + 'atlas' => 57600, + 'blender' => 57600, + 'chromium-browser-stable' => 172800, + 'clang' => 172800, + 'cross-gcc' => 115200, + 'gcc' => 115200, + 'itk' => 115200, + 'java-1.8.0-openjdk' => 172800, + 'java-17-openjdk' => 172800, + 'java-21-openjdk' => 172800, + 'java-latest-openjdk' => 172800, + 'kernel' => 115200, + 'libreoffice' => 432000, + 'llvm' => 115200, + 'llvm17-suite' => 115200, + 'llvm19-suite' => 115200, + 'openfoam' => 115200, + 'paraview' => 115200, + 'qgis' => 57600, + 'qtwebengine5' => 115200, + 'qtwebengine6' => 172800, + 'rust' => 180000, + 'salome' => 57600, + 'vtk' => 57600, + 'webkit' => 57600, + 'webkit2' => 115200, + 'wrapitk' => 115200, + 'rocm-llvm' => 70000, + } + + $allow_network_access = [ + 'libguestfs', # Needs access to the configured mirrors + ] + + file { "/etc/iurt/build/${distribution}.conf": owner => $build_login, group => $build_login, - content => template("buildsystem/iurt/$distribution.conf") + content => template("buildsystem/iurt.conf") } } diff --git a/modules/buildsystem/manifests/iurt/packages.pp b/modules/buildsystem/manifests/iurt/packages.pp index 448b4a89..e814b7c2 100644 --- a/modules/buildsystem/manifests/iurt/packages.pp +++ b/modules/buildsystem/manifests/iurt/packages.pp @@ -1,3 +1,3 @@ class buildsystem::iurt::packages { package { 'iurt': } -} +} diff --git a/modules/buildsystem/manifests/iurt/upload.pp b/modules/buildsystem/manifests/iurt/upload.pp index 8c9026c0..5417d36e 100644 --- a/modules/buildsystem/manifests/iurt/upload.pp +++ b/modules/buildsystem/manifests/iurt/upload.pp @@ -1,8 +1,7 @@ class buildsystem::iurt::upload { - $login = $buildsystem::iurt::user::login - $homedir = $buildsystem::iurt::user::homedir - $sched_login = $buildsystem::scheduler::login - + include buildsystem::var::iurt + include buildsystem::var::webstatus + include buildsystem::var::repository file { '/etc/iurt/upload.conf': require => File['/etc/iurt'], content => template('buildsystem/upload.conf'), diff --git a/modules/buildsystem/manifests/iurt/user.pp b/modules/buildsystem/manifests/iurt/user.pp index b169ee2d..a93ac7e7 100644 --- a/modules/buildsystem/manifests/iurt/user.pp +++ b/modules/buildsystem/manifests/iurt/user.pp @@ -1,10 +1,8 @@ class buildsystem::iurt::user { - $login = 'iurt' - $homedir = "/home/$login" + include buildsystem::var::iurt - buildsystem::sshuser { $login: - homedir => $homedir, - comment => 'System user used to run build bots', + buildsystem::sshuser { $buildsystem::var::iurt::login: + homedir => $buildsystem::var::iurt::homedir, } file { '/etc/iurt': diff --git a/modules/buildsystem/manifests/mainnode.pp b/modules/buildsystem/manifests/mainnode.pp index 45dff1dd..01de764f 100644 --- a/modules/buildsystem/manifests/mainnode.pp +++ b/modules/buildsystem/manifests/mainnode.pp @@ -1,5 +1,7 @@ class buildsystem::mainnode { - include buildsystem::base + include buildsystem::var::repository + include buildsystem::var::scheduler + include buildsystem::var::distros include buildsystem::iurt::user include buildsystem::scheduler include buildsystem::gatherer @@ -7,27 +9,15 @@ class buildsystem::mainnode { include buildsystem::signbot include buildsystem::youri_submit include buildsystem::sshkeys + include buildsystem::distros - $sched_login = $buildsystem::scheduler::var::login - $sched_home_dir = $buildsystem::scheduler::var::homedir - - $build_login = $buildsystem::iurt::user::login - - ssh::auth::client { $sched_login: } - - ssh::auth::server { [$sched_login, $build_login]: } - - $mirror_root = '/distrib/mirror' - apache::vhost::other_app { "repository.$::domain": - vhost_file => 'buildsystem/vhost_repository.conf', - } - - buildsystem::media_cfg { "cauldron i586": - distro => 'cauldron', - arch => 'i586', + sshkeys::set_client_key_pair { $buildsystem::var::scheduler::login: + home => $buildsystem::var::scheduler::homedir, + user => $buildsystem::var::scheduler::login, } - buildsystem::media_cfg { "cauldron x86_64": - distro => 'cauldron', - arch => 'x86_64', + sshkeys::set_authorized_keys { 'scheduler-allow-scheduler': + keyname => $buildsystem::var::scheduler::login, + home => $buildsystem::var::scheduler::homedir, + user => $buildsystem::var::scheduler::login, } } diff --git a/modules/buildsystem/manifests/maintdb.pp b/modules/buildsystem/manifests/maintdb.pp index 02782271..5a961b63 100644 --- a/modules/buildsystem/manifests/maintdb.pp +++ b/modules/buildsystem/manifests/maintdb.pp @@ -1,31 +1,27 @@ class buildsystem::maintdb { + include buildsystem::var::maintdb + include buildsystem::var::groups + include buildsystem::var::webstatus include sudo - $login = 'maintdb' - $homedir = '/var/lib/maintdb' - $dbdir = "$homedir/db" - $binpath = '/usr/local/sbin/maintdb' - $dump = '/var/www/bs/data/maintdb.txt' - $unmaintained = '/var/www/bs/data/unmaintained.txt' - user { $login: - comment => 'Maintainers database', - home => $homedir, + user { $buildsystem::var::maintdb::login: + home => $buildsystem::var::maintdb::homedir, } - file { [$homedir,$dbdir]: + file { [$buildsystem::var::maintdb::homedir,$buildsystem::var::maintdb::dbdir]: ensure => directory, - owner => $login, - group => $login, + owner => $buildsystem::var::maintdb::login, + group => $buildsystem::var::maintdb::login, mode => '0711', - require => User[$login], + require => User[$buildsystem::var::maintdb::login], } - file { $binpath: + file { $buildsystem::var::maintdb::binpath: mode => '0755', content => template('buildsystem/maintdb/maintdb.bin') } - mga-common::local_script { 'wrapper.maintdb': + mga_common::local_script { 'wrapper.maintdb': content => template('buildsystem/maintdb/wrapper.maintdb') } @@ -33,22 +29,30 @@ class buildsystem::maintdb { content => template('buildsystem/maintdb/sudoers.maintdb') } - file { [$dump,"$dump.new", - $unmaintained,"$unmaintained.new"]: - owner => $login, -# TODO uncomment once the situation with pkgsubmit module is cleared ( ie, maintdb depend on it ) -# require => File['/var/www/bs/data'], + file { [$buildsystem::var::maintdb::dump, + "${buildsystem::var::maintdb::dump}.new", + $buildsystem::var::maintdb::unmaintained, + "${buildsystem::var::maintdb::unmaintained}.new"]: + owner => $buildsystem::var::maintdb::login, + require => File["${buildsystem::var::webstatus::location}/data"], } cron { 'update maintdb export': - user => $login, - command => "$binpath root get > $dump.new; cp -f $dump.new $dump; grep ' nobody\$' $dump | sed 's/ nobody\$//' > $unmaintained.new; cp -f $unmaintained.new $unmaintained", + user => $buildsystem::var::maintdb::login, + command => "${buildsystem::var::maintdb::binpath} root get > ${buildsystem::var::maintdb::dump}.new; cp -f ${buildsystem::var::maintdb::dump}.new ${buildsystem::var::maintdb::dump}; grep ' nobody\$' ${buildsystem::var::maintdb::dump} | sed 's/ nobody\$//' > ${buildsystem::var::maintdb::unmaintained}.new; cp -f ${buildsystem::var::maintdb::unmaintained}.new ${buildsystem::var::maintdb::unmaintained}", minute => '*/30', - require => User[$login], + require => User[$buildsystem::var::maintdb::login], } - apache::vhost::base { "maintdb.$::domain": - location => $dbdir, + apache::vhost::base { $buildsystem::var::maintdb::hostname: + location => $buildsystem::var::maintdb::dbdir, + content => template('buildsystem/maintdb/vhost_maintdb.conf'), + } + + apache::vhost::base { "ssl_${buildsystem::var::maintdb::hostname}": + use_ssl => true, + vhost => $buildsystem::var::maintdb::hostname, + location => $buildsystem::var::maintdb::dbdir, content => template('buildsystem/maintdb/vhost_maintdb.conf'), } } diff --git a/modules/buildsystem/manifests/media_cfg.pp b/modules/buildsystem/manifests/media_cfg.pp index c6de0d35..77fcc8fd 100644 --- a/modules/buildsystem/manifests/media_cfg.pp +++ b/modules/buildsystem/manifests/media_cfg.pp @@ -1,10 +1,11 @@ -define buildsystem::media_cfg($distro, $arch, $templatefile = 'buildsystem/media.cfg') { - include buildsystem::scheduler::var +define buildsystem::media_cfg($distro_name, $arch, $templatefile = 'buildsystem/media.cfg') { + include buildsystem::var::repository + include buildsystem::var::scheduler include buildsystem::repository - - file { "${buildsystem::repository::dir}/distrib/${distro}/${arch}/media/media_info/media.cfg": - owner => $buildsystem::base::sched_login, - group => $buildsystem::base::sched_login, - content => template($templatefile), + + file { "${buildsystem::var::repository::bootstrap_reporoot}/${distro_name}/${arch}/media/media_info/media.cfg": + owner => $buildsystem::var::scheduler::login, + group => $buildsystem::var::scheduler::login, + content => template($templatefile), } } diff --git a/modules/buildsystem/manifests/mgarepo.pp b/modules/buildsystem/manifests/mgarepo.pp index 70b71202..14e11e1a 100644 --- a/modules/buildsystem/manifests/mgarepo.pp +++ b/modules/buildsystem/manifests/mgarepo.pp @@ -1,72 +1,36 @@ class buildsystem::mgarepo { - include buildsystem::scheduler::var - $sched_login = $buildsystem::scheduler::var::login - $sched_home_dir = $buildsystem::scheduler::var::homedir + include buildsystem::var::scheduler + include buildsystem::var::distros + include buildsystem::var::groups + include buildsystem::var::binrepo + include buildsystem::create_upload_dir + $sched_login = $buildsystem::var::scheduler::login + $sched_home_dir = $buildsystem::var::scheduler::homedir package { ['mgarepo','rpm-build']: } - file { - '/etc/mgarepo.conf': content => template('buildsystem/mgarepo.conf'); - '/etc/repsys.conf': content => template('buildsystem/mgarepo.conf'); + file { '/etc/mgarepo.conf': + content => template('buildsystem/mgarepo.conf'), } - file { "$sched_home_dir/repsys": + file { "${sched_home_dir}/repsys": ensure => 'directory', owner => $sched_login, require => File[$sched_home_dir], } - file { ["$sched_home_dir/repsys/tmp", "$sched_home_dir/repsys/srpms"]: + file { ["${sched_home_dir}/repsys/tmp", "${sched_home_dir}/repsys/srpms"]: ensure => 'directory', owner => $sched_login, - group => $buildsystem::base::packagers_group, + group => $buildsystem::var::groups::packagers, mode => '1775', - require => File["$sched_home_dir/repsys"], + require => File["${sched_home_dir}/repsys"], } - # FIXME: disabled temporarly as upload dir is a symlink to /var/lib/repsys/uploads - #file { "$sched_home_dir/uploads": + # FIXME: disabled temporarily as upload dir is a symlink to /var/lib/repsys/uploads + #file { "${sched_home_dir}/uploads": # ensure => "directory", # owner => $sched_login, # require => File[$sched_home_dir], #} - - #FIXME This config information should be moved out of this class - $releases = { - 'cauldron' => { - 'core' => ['release','updates_testing','backports_testing','backports','updates'], - 'nonfree' => ['release','updates_testing','backports_testing','backports','updates'], - 'tainted' => ['release','updates_testing','backports_testing','backports','updates'], - }, - '1' => { - 'core' => ['release','updates_testing','backports_testing','backports','updates'], - 'nonfree' => ['release','updates_testing','backports_testing','backports','updates'], - 'tainted' => ['release','updates_testing','backports_testing','backports','updates'], - }, - '2' => { - 'core' => ['release','updates_testing','backports_testing','backports','updates'], - 'nonfree' => ['release','updates_testing','backports_testing','backports','updates'], - 'tainted' => ['release','updates_testing','backports_testing','backports','updates'], - }, - 'infra_1' => { - 'infra' => ['release'] - }, - 'infra_2' => { - 'infra' => ['release'] - }, - } - - import 'create_upload_dir.rb' - create_upload_dir { "$sched_home_dir/uploads": - owner => $sched_login, - group => $sched_login, - releases => $releases, - } - - tidy { "$sched_home_dir/uploads": - type => 'ctime', - recurse => true, - age => '2w', - } - } diff --git a/modules/buildsystem/manifests/pkgsubmit.pp b/modules/buildsystem/manifests/pkgsubmit.pp deleted file mode 100644 index ba575cd2..00000000 --- a/modules/buildsystem/manifests/pkgsubmit.pp +++ /dev/null @@ -1,19 +0,0 @@ -class buildsystem::pkgsubmit { - include buildsystem::scheduler::var - $sched_home_dir = $buildsystem::scheduler::var::homedir - - $location = '/var/www/bs' - file { [$location,"$location/data"]: - ensure => directory, - } - - apache::vhost::base { "pkgsubmit.$::domain": - aliases => { '/uploads' => "$sched_home_dir/uploads" }, - location => $location, - content => template('buildsystem/vhost_pkgsubmit.conf'), - } - - subversion::snapshot { $location: - source => "svn://svn.$::domain/soft/buildsystem/web/", - } -} diff --git a/modules/buildsystem/manifests/release.pp b/modules/buildsystem/manifests/release.pp index 1682d71b..d9feac8e 100644 --- a/modules/buildsystem/manifests/release.pp +++ b/modules/buildsystem/manifests/release.pp @@ -1,7 +1,5 @@ class buildsystem::release { - subversion::snapshot { '/root/release': - source => "svn://svn.$::domain/soft/release/trunk/", + git::snapshot { '/root/release': + source => "git://git.${::domain}/software/infrastructure/release", } - - package { 'hardlink': } } diff --git a/modules/buildsystem/manifests/repoctl.pp b/modules/buildsystem/manifests/repoctl.pp index 024a4a09..8d44e52c 100644 --- a/modules/buildsystem/manifests/repoctl.pp +++ b/modules/buildsystem/manifests/repoctl.pp @@ -1,16 +1,11 @@ class buildsystem::repoctl { - include buildsystem::config + include buildsystem::var::distros + include buildsystem::var::repository - $distroreleases = [$buildsystem::config::dev_distros, - $buildsystem::config::stable_distros] - $distrosections = $buildsystem::config::distrosections - $sectionsrepos = $buildsystem::config::sectionsrepos - $arches = $buildsystem::config::architectures + package{ 'repoctl': } - package{ 'repoctl': } - - file { '/etc/repoctl.conf': - content => template('buildsystem/repoctl.conf'), - require => Package['repoctl'], - } + file { '/etc/repoctl.conf': + content => template('buildsystem/repoctl.conf'), + require => Package['repoctl'], + } } diff --git a/modules/buildsystem/manifests/repository.pp b/modules/buildsystem/manifests/repository.pp index 19e69eff..dda90eb2 100644 --- a/modules/buildsystem/manifests/repository.pp +++ b/modules/buildsystem/manifests/repository.pp @@ -1,6 +1,11 @@ class buildsystem::repository { - $dir = '/distrib/bootstrap' - file { $dir: + include buildsystem::var::repository + file { [ $buildsystem::var::repository::bootstrap_root, + $buildsystem::var::repository::bootstrap_reporoot ] : ensure => directory, - } + } + + apache::vhost::other_app { $buildsystem::var::repository::hostname: + vhost_file => 'buildsystem/vhost_repository.conf', + } } diff --git a/modules/buildsystem/manifests/scheduler.pp b/modules/buildsystem/manifests/scheduler.pp index 6facc8e9..53b248fc 100644 --- a/modules/buildsystem/manifests/scheduler.pp +++ b/modules/buildsystem/manifests/scheduler.pp @@ -1,28 +1,57 @@ class buildsystem::scheduler { - # until ulri is splitted from main iurt rpm - include ssh::auth + # until ulri is split from main iurt rpm include buildsystem::iurt::packages include buildsystem::iurt::upload - include buildsystem::scheduler::var + include buildsystem::var::scheduler - $login = $buildsystem::scheduler::var::login - $homedir = $buildsystem::scheduler::var::homedir - $logdir = $buildsystem::scheduler::var::logdir + $login = $buildsystem::var::scheduler::login + $homedir = $buildsystem::var::scheduler::homedir + $logdir = $buildsystem::var::scheduler::logdir buildsystem::sshuser { $login: homedir => $homedir, - comment => 'System user used to schedule builds', } - file { $logdir: - ensure => directory, - mode => 0755, - owner => $login, - } + file { $logdir: + ensure => directory, + mode => '0755', + owner => $login, + } + + cron { 'dispatch jobs': + user => $login, + command => "EMI_LOG_FILE=${logdir}/emi.log ULRI_LOG_FILE=${logdir}/ulri.log ulri; EMI_LOG_FILE=${logdir}/emi.log emi", + minute => '*', + } - cron { 'dispatch jobs': - user => $login, - command => "ULRI_LOG_FILE=$logdir/ulri.log ulri; EMI_LOG_FILE=$logdir/emi.log emi", - minute => '*', - } + if ($buildsystem::var::scheduler::clean_uploads_logs_age != 0) { + cron { 'clean uploads logs': + user => $login, + # Delete old upload logs + command => sprintf("/usr/bin/find %s/uploads -ignore_readdir_race -xdev -depth -type f -ctime +%d -delete", shellquote($homedir), shellquote($buildsystem::var::scheduler::clean_uploads_logs_age)), + hour => '*/4', + minute => '51', + } + cron { 'clean uploads dirs': + user => $login, + # Remove old empty uploads directories. This will take several + # passes (over several weeks) to delete a directory hierarchy + # because it is looking at ctime instead of mtime, which resets + # every time a file/directory underneath it is deleted. + # Directories don't take much space, so this shouldn't be a + # real issue. + command => sprintf("/usr/bin/find %s/uploads -ignore_readdir_race -mindepth 5 -xdev -depth -type d -ctime +%d -empty -delete", shellquote($homedir), shellquote($buildsystem::var::scheduler::clean_uploads_logs_age)), + hour => '*/4', + minute => '53', + } + } + if ($buildsystem::var::scheduler::clean_uploads_packages_age != 0) { + cron { 'clean uploads packages': + user => $login, + # Delete old upload RPMs + command => sprintf("/usr/bin/find %s/uploads -ignore_readdir_race -xdev -depth -type f -name '*.rpm' -ctime +%d -delete", shellquote($homedir), shellquote($buildsystem::var::scheduler::clean_uploads_packages_age)), + hour => '*/4', + minute => '52', + } + } } diff --git a/modules/buildsystem/manifests/scheduler/var.pp b/modules/buildsystem/manifests/scheduler/var.pp deleted file mode 100644 index e89ef634..00000000 --- a/modules/buildsystem/manifests/scheduler/var.pp +++ /dev/null @@ -1,5 +0,0 @@ -class buildsystem::scheduler::var { - $login = 'schedbot' - $homedir = "/var/lib/$login" - $logdir = "/var/log/$login" -} diff --git a/modules/buildsystem/manifests/signbot.pp b/modules/buildsystem/manifests/signbot.pp index 5e5db5d9..60c7c318 100644 --- a/modules/buildsystem/manifests/signbot.pp +++ b/modules/buildsystem/manifests/signbot.pp @@ -1,38 +1,31 @@ class buildsystem::signbot { - include buildsystem::scheduler::var - $login = 'signbot' - $home_dir = "/var/lib/$login" - $sign_keydir = "$home_dir/keys" - # FIXME: maybe keyid should be defined at an other place - $keyid = '80420F66' - $sched_login = $buildsystem::scheduler::var::login + include buildsystem::var::scheduler + include buildsystem::var::signbot + $sched_login = $buildsystem::var::scheduler::login - sshuser { $login: - homedir => $home_dir, - comment => 'System user used to sign packages', + sshuser { $buildsystem::var::signbot::login: + homedir => $buildsystem::var::signbot::home_dir, groups => [$sched_login], } gnupg::keys{ 'packages': - email => "packages@$::domain", - #FIXME there should be a variable somewhere to change - # the name of the distribution - key_name => 'Mageia Packages', - login => $login, - batchdir => "$home_dir/batches", - keydir => $sign_keydir, + email => $buildsystem::var::signbot::keyemail, + key_name => $buildsystem::var::signbot::keyname, + login => $buildsystem::var::signbot::login, + batchdir => "${buildsystem::var::signbot::home_dir}/batches", + keydir => $buildsystem::var::signbot::sign_keydir, } sudo::sudoers_config { 'signpackage': content => template('buildsystem/signbot/sudoers.signpackage') } - file { "$home_dir/.rpmmacros": - content => template('buildsystem/signbot/signbot-rpmmacros') + file { "${home_dir}/.rpmmacros": + source => 'puppet:///modules/buildsystem/signbot/signbot-rpmmacros', } - mga-common::local_script { - 'sign-check-package': content => template('buildsystem/signbot/sign-check-package'); - 'mga-signpackage': content => template('buildsystem/signbot/mga-signpackage'); + mga_common::local_script { + 'sign-check-package': source => 'puppet:///modules/buildsystem/signbot/sign-check-package'; + 'mga-signpackage': source => 'puppet:///modules/buildsystem/signbot/mga-signpackage'; } } diff --git a/modules/buildsystem/manifests/sshkeys.pp b/modules/buildsystem/manifests/sshkeys.pp index bea2959f..5a1b2900 100644 --- a/modules/buildsystem/manifests/sshkeys.pp +++ b/modules/buildsystem/manifests/sshkeys.pp @@ -1,13 +1,5 @@ class buildsystem::sshkeys { - include ssh::auth - include buildsystem::scheduler::var - include buildsystem::iurt::user + include buildsystem::var::scheduler - ssh::auth::key { $buildsystem::scheduler::var::login: - home => $buildsystem::scheduler::var::homedir, - } - - ssh::auth::key { $buildsystem::iurt::user::login: - home => $buildsystem::iurt::user::homedir - } + sshkeys::create_key { $buildsystem::var::scheduler::login: } } diff --git a/modules/buildsystem/manifests/sshuser.pp b/modules/buildsystem/manifests/sshuser.pp index 0a1cd176..5cad97ad 100644 --- a/modules/buildsystem/manifests/sshuser.pp +++ b/modules/buildsystem/manifests/sshuser.pp @@ -1,5 +1,5 @@ # $groups: array of secondary groups (only local groups, no ldap) -define buildsystem::sshuser($homedir, $comment, $groups = []) { +define buildsystem::sshuser($homedir, $comment = undef, $groups = []) { group { $name: } user { $name: @@ -9,13 +9,13 @@ define buildsystem::sshuser($homedir, $comment, $groups = []) { gid => $name, groups => $groups, shell => '/bin/bash', - notify => Exec["unlock $name"], + notify => Exec["unlock ${name}"], require => Group[$title], } # set password to * to unlock the account but forbid login through login - exec { "unlock $name": - command => "usermod -p '*' $name", + exec { "unlock ${name}": + command => "usermod -p '*' ${name}", refreshonly => true, } @@ -26,7 +26,7 @@ define buildsystem::sshuser($homedir, $comment, $groups = []) { require => User[$name], } - file { "$homedir/.ssh": + file { "${homedir}/.ssh": ensure => directory, mode => '0600', owner => $name, diff --git a/modules/buildsystem/manifests/var/binrepo.pp b/modules/buildsystem/manifests/var/binrepo.pp new file mode 100644 index 00000000..1431ed25 --- /dev/null +++ b/modules/buildsystem/manifests/var/binrepo.pp @@ -0,0 +1,15 @@ +# $uploadmail_from: +# from who will be sent the binrepo upload email notifications +# $uploadmail_to: +# where binrepo email notifications are sent +class buildsystem::var::binrepo( + $hostname = "binrepo.${::domain}", + $login = 'binrepo', + $homedir = '/var/lib/binrepo', + $uploadmail_from, + $uploadmail_to +) { + $repodir = "${homedir}/data" + $uploadinfosdir = "${homedir}/infos" + $uploadbinpath = '/usr/local/bin/upload-bin' +} diff --git a/modules/buildsystem/manifests/var/distros.pp b/modules/buildsystem/manifests/var/distros.pp new file mode 100644 index 00000000..9e45e2c2 --- /dev/null +++ b/modules/buildsystem/manifests/var/distros.pp @@ -0,0 +1,126 @@ +# $default_distro: +# the name of the default distribution +# $repo_allow_from_ips: +# $repo_allow_from_domains: +# list of IP or domains allowed to access the repository. If you don't want to +# filter allowed IPs, don't those values. +# $distros: +# a hash variable containing distributions information indexed by +# distribution name. Each distribution is itself an hash containing +# the following infos: +# { +# # the 'cauldron' distribution +# 'cauldron' => { +# # list of arch supported by 'cauldron' +# 'arch' => [ 'i586', 'x86_64' ], +# # Set this if you don't want media.cfg to be generated +# 'no_media_cfg_update' => true, +# 'medias' => { +# # the 'core' media +# 'core' => { +# 'repos' => { +# # the 'release' repo in the 'core' media +# 'release' => { +# 'media_type' => [ 'release' ], +# 'noauto' => '1', +# # the 'release' repo should be listed first in media.cfg +# 'order' => 0, +# }, +# # the 'updates' repo +# 'updates' => { +# 'media_type' => [ 'updates' ], +# 'noauto' => '1', +# # the 'updates' repo requires the 'release' repo +# 'requires' => [ 'release' ], +# # the 'updates' repo should be listed after 'release' in media.cfg +# 'order' => 1, +# }, +# }, +# # media_type for media.cfg +# 'media_type' => [ 'official', 'free' ], +# # if noauto is set to '1' either in medias or repos, +# # the option will be added to media.cfg +# 'noauto' => '1', +# # list 'core' first in media.cfg +# 'order' => 0, +# }, +# # the 'non-free' media +# 'non-free' => { +# 'repos' => { +# ... +# }, +# 'media_type' => [ 'official', 'non-free' ], +# # the 'non-free' media requires the 'core' media +# 'requires' => [ 'core' ], +# # list 'non-free' second +# 'order' => 1, +# } +# }, +# # the list of media used by iurt to build the chroots +# 'base_medias' => [ 'core/release' ], +# # optionally, a media.cfg template file can be specified, if +# # the default one should not be used +# 'tmpl_media.cfg' => 'buildsystem/something', +# # branch is Devel or Official. Used in media.cfg. +# 'branch' => 'Devel', +# # Version of the distribution +# 'version' => '3', +# # SVN Urls allowed to submit +# 'submit_allowed' => 'svn://svn.something/svn/packages/cauldron', +# # rpm macros to set when build source package +# 'macros' => { +# 'distsuffix' => '.mga', +# 'distribution' => 'Mageia', +# 'vendor' => 'Mageia.Org', +# }, +# # set this if the distro is not mirrored. This is used to add +# # an Alias in the vhost. +# 'no_mirror' => true, +# Optionally, the distribution can be based on the repos from an other +# distribution. In this example we're saying that the distribution is +# based on 2/core/release and 2/core/updates. +# 'based_on' => { +# '2' => { +# 'core' => [ 'release', 'updates' ], +# }, +# }, +# 'youri' => { +# # Configuration for youri-upload +# 'upload' => { +# # list of enabled checks, actions and posts +# 'targets' => { +# 'checks' => [ +# ... +# ], +# 'actions' => [ +# ... +# ], +# 'posts' => [ +# ... +# ], +# }, +# 'checks' => { +# # rpmlint checks options +# 'rpmlint' => { +# 'config' => '/usr/share/rpmlint/config', +# 'path' => ''/usr/bin/rpmlint', +# }, +# }, +# # options for actions +# 'actions' => { +# ... +# }, +# }, +# # Configuration for youri-todo +# 'todo' => { +# ... +# }, +# }, +# }, +# } +class buildsystem::var::distros( + $default_distro, + $repo_allow_from_ips, + $repo_allow_from_domains, + $distros, +) { } diff --git a/modules/buildsystem/manifests/var/groups.pp b/modules/buildsystem/manifests/var/groups.pp new file mode 100644 index 00000000..c0b2c917 --- /dev/null +++ b/modules/buildsystem/manifests/var/groups.pp @@ -0,0 +1,9 @@ +# $packagers: +# name of packagers group, who should be allowed to submit packages +# $packagers_committers: +# name of group of users who are allowed to commit on packages +class buildsystem::var::groups( + $packagers, + $packagers_committers +) { +} diff --git a/modules/buildsystem/manifests/var/iurt.pp b/modules/buildsystem/manifests/var/iurt.pp new file mode 100644 index 00000000..fb65a160 --- /dev/null +++ b/modules/buildsystem/manifests/var/iurt.pp @@ -0,0 +1,5 @@ +class buildsystem::var::iurt( + $login = 'iurt', + $homedir = '/home/iurt', + $timeout_multiplier = 1, +) { } diff --git a/modules/buildsystem/manifests/var/maintdb.pp b/modules/buildsystem/manifests/var/maintdb.pp new file mode 100644 index 00000000..e0079e40 --- /dev/null +++ b/modules/buildsystem/manifests/var/maintdb.pp @@ -0,0 +1,11 @@ +class buildsystem::var::maintdb( + $hostname = "maintdb.${::domain}", + $login = 'maintdb', + $homedir = '/var/lib/maintdb' +) { + include buildsystem::var::webstatus + $dbdir = "${homedir}/db" + $binpath = '/usr/local/sbin/maintdb' + $dump = "${buildsystem::var::webstatus::location}/data/maintdb.txt" + $unmaintained = "${buildsystem::var::webstatus::location}/data/unmaintained.txt" +} diff --git a/modules/buildsystem/manifests/var/mgarepo.pp b/modules/buildsystem/manifests/var/mgarepo.pp new file mode 100644 index 00000000..9099c7ee --- /dev/null +++ b/modules/buildsystem/manifests/var/mgarepo.pp @@ -0,0 +1,22 @@ +# $submit_host: +# hostname used to submit packages +# $svn_hostname: +# hostname of the svn server used for packages +# $svn_root_packages: +# svn root url of the svn repository for packages +# $svn_root_packages_ssh: +# svn+ssh root url of the svn repository for packages +# $oldurl: +# svn url where the import logs of the rpm are stored +# $conf: +# $conf{'global'} is a has table of values used in mgarepo.conf in +# the [global] section +class buildsystem::var::mgarepo( + $submit_host, + $svn_hostname, + $svn_root_packages, + $svn_root_packages_ssh, + $oldurl, + $conf +) { +} diff --git a/modules/buildsystem/manifests/var/repository.pp b/modules/buildsystem/manifests/var/repository.pp new file mode 100644 index 00000000..0ea1058c --- /dev/null +++ b/modules/buildsystem/manifests/var/repository.pp @@ -0,0 +1,9 @@ +class buildsystem::var::repository( + $hostname = "repository.${::domain}", + $bootstrap_root = '/distrib/bootstrap', + $mirror_root = '/distrib/mirror', + $distribdir = 'distrib' +) { + $bootstrap_reporoot = "${bootstrap_root}/${distribdir}" + $mirror_reporoot = "${mirror_root}/${distribdir}" +} diff --git a/modules/buildsystem/manifests/var/scheduler.pp b/modules/buildsystem/manifests/var/scheduler.pp new file mode 100644 index 00000000..b431594c --- /dev/null +++ b/modules/buildsystem/manifests/var/scheduler.pp @@ -0,0 +1,31 @@ +# $admin_mail: +# the email address from which the build failure notifications +# will be sent +# $pkg_uphost: +# hostname of the server where submitted packages are uploaded +# $build_nodes: +# a hash containing available build nodes indexed by architecture +# $build_nodes_aliases: +# a hash containing build nodes indexed by their alias +# $build_src_node: +# hostname of the server building the initial src.rpm +# $clean_uploads_logs_age: +# old logs are cleaned when they are older than some amount of days. +# You can define this amount of time using this variable. Set it to +# 14 for two weeks, 2 for two days, or 0 if you don't want to +# clean old logs at all +# $clean_uploads_packages_age: +# same as $clean_uploads_logs_age but for old RPMs +class buildsystem::var::scheduler( + $admin_mail = "root@${::domain}", + $pkg_uphost = "pkgsubmit.${::domain}", + $build_nodes, + $build_nodes_aliases = {}, + $build_src_node, + $clean_uploads_logs_age = 14, + $clean_uploads_packages_age = 7 +){ + $login = 'schedbot' + $homedir = "/var/lib/${login}" + $logdir = "/var/log/${login}" +} diff --git a/modules/buildsystem/manifests/var/signbot.pp b/modules/buildsystem/manifests/var/signbot.pp new file mode 100644 index 00000000..7d92a324 --- /dev/null +++ b/modules/buildsystem/manifests/var/signbot.pp @@ -0,0 +1,15 @@ +# $keyid: +# the key id of the gnupg key used to sign packages +# $keyemail: +# email address of the key used to sign packages +# $keyname: +# name of the key used to sign packages +class buildsystem::var::signbot( + $keyid, + $keyemail, + $keyname +) { + $login = 'signbot' + $home_dir = "/var/lib/${login}" + $sign_keydir = "${home_dir}/keys" +} diff --git a/modules/buildsystem/manifests/var/webstatus.pp b/modules/buildsystem/manifests/var/webstatus.pp new file mode 100644 index 00000000..21f8d59f --- /dev/null +++ b/modules/buildsystem/manifests/var/webstatus.pp @@ -0,0 +1,25 @@ +# $git_url: +# git url where the sources of webstatus are located +# $hostname: +# vhost name of the webstatus page +# $location: +# path of the directory where the webstatus files are located +# $package_commit_url: +# url to view a commit on a package. %d is replaced by the commit id. +# $max_modified: +# how much history should we display, in days +# $theme_name: +# name of the webstatus theme +# $themes_dir: +# path of the directory where the themes are located. If you want +# to use a theme not included in webstatus, you need to change this. +class buildsystem::var::webstatus( + $git_url = "git://git.${::domain}/web/pkgsubmit", + $hostname = "pkgsubmit.${::domain}", + $location = '/var/www/bs', + $package_commit_url, + $max_modified = '2', + $theme_name = 'mageia', + $themes_dir = '/var/www/bs/themes/' +) { +} diff --git a/modules/buildsystem/manifests/var/youri.pp b/modules/buildsystem/manifests/var/youri.pp new file mode 100644 index 00000000..f20b6c7b --- /dev/null +++ b/modules/buildsystem/manifests/var/youri.pp @@ -0,0 +1,401 @@ +# The youri configuration files are created using information from 3 +# different hash variables : +# - the $youri_conf_default variable defined in this class, containing +# the default configuration for youri. It contains the repository +# configuration, and the definitions of the checks, actions and posts. +# - the $youri_conf parameter passed to this class. The values defined +# in this hash override the values defined in the default configuration. +# - for each distribution defined in the hash variable $distros from +# var::buildsystem::distros the hash defined in index 'youri' contains +# some distro specific options for youri checks, actions or posts. It +# also contains for each distribution the list of active checks, +# actions and posts. +# +# Each of those variables contain the configuration for youri submit-todo +# (in index 'todo') and youri submit-upload (in index 'upload') +# +# +# Parameters : +# $tmpl_youri_upload_conf: +# template file for youri submit-upload.conf +# $tmpl_youri_todo_conf: +# template file for youri submit-todo.conf +# $packages_archivedir: +# the directory where youri will archive old packages when they are +# replaced by a new version +# $youri_conf: +# a hash containing the youri configuration +class buildsystem::var::youri( + $tmpl_youri_upload_conf = 'buildsystem/youri/submit.conf', + $tmpl_youri_todo_conf = 'buildsystem/youri/submit.conf', + $packages_archivedir, + $youri_conf = {} +) { + include buildsystem::var::repository + include buildsystem::var::mgarepo + include buildsystem::var::distros + include buildsystem::var::signbot + include buildsystem::var::scheduler + + $check_tag = { 'class' => 'Youri::Submit::Check::Tag', } + $check_recency = { 'class' => 'Youri::Submit::Check::Recency', } + $check_queue_recency = { 'class' => 'Youri::Submit::Check::Queue_recency', } + $check_host = { + 'class' => 'Youri::Submit::Check::Host', + 'options' => { + 'host_file' => '/etc/youri/host.conf', + }, + } + $check_rpmlint = { 'class' => 'Youri::Submit::Check::Rpmlint', } + $check_acl = { + 'class' => 'Youri::Submit::Check::ACL', + 'options' => { + 'acl_file' => '/etc/youri/acl.conf', + }, + } + $check_source = { 'class' => 'Youri::Submit::Check::Source', } + $check_version = { + 'class' => 'Youri::Submit::Check::Version', + 'options' => {}, + } + + $youri_conf_default = { + 'upload' => { + 'repository' => { + 'class' => 'Youri::Repository::Mageia', + 'options' => { + 'install_root' => $buildsystem::var::repository::bootstrap_reporoot, + 'upload_root' => '$home/uploads/', + 'archive_root' => $packages_archivedir, + 'upload_state' => 'queue', + 'queue' => 'queue', + 'noarch' => 'x86_64', + 'svn' => "${buildsystem::var::mgarepo::svn_root_packages_ssh}/${buildsystem::var::distros::default_distro}", + }, + }, + 'checks' => { + 'tag' => $check_tag, + 'recency' => $check_recency, + 'queue_recency' => $check_queue_recency, + 'host' => $check_host, + 'section' => { + 'class' => 'Youri::Submit::Check::Section', + }, + 'rpmlint' => $check_rpmlint, + 'svn' => { + 'class' => 'Youri::Submit::Check::SVN', + }, + 'acl' => $check_acl, + 'history' => { + 'class' => 'Youri::Submit::Check::History', + }, + 'source' => $check_source, + 'precedence' => { + 'class' => 'Youri::Submit::Check::Precedence', + 'options' => { + 'target' => $buildsystem::var::distros::default_distro, + }, + }, + 'version' => $check_version, + }, + 'actions' => { + 'install' => { + 'class' => 'Youri::Submit::Action::Install', + }, + 'markrelease' => { + 'class' => 'Youri::Submit::Action::Markrelease', + }, + 'link' => { + 'class' => 'Youri::Submit::Action::Link', + }, + 'archive' => { + 'class' => 'Youri::Submit::Action::Archive', + }, + 'clean' => { + 'class' => 'Youri::Submit::Action::Clean', + }, + 'sign' => { + 'class' => 'Youri::Submit::Action::Sign', + 'options' => { + 'signuser' => $buildsystem::var::signbot::login, + 'path' => $buildsystem::var::signbot::sign_keydir, + 'name' => $buildsystem::var::signbot::keyid, + 'signscript' => '/usr/local/bin/sign-check-package', + }, + }, + 'unpack_gfxboot_theme' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'mageia-gfxboot-theme', + 'source_subdir' => '/usr/share/gfxboot/themes/Mageia/install/', + 'dest_directory' => 'isolinux', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_meta_task' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'meta-task', + 'source_subdir' => '/usr/share/meta-task', + 'dest_directory' => 'media/media_info', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_images' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-images', + 'source_subdir' => '/usr/lib*/drakx-installer-images', + 'dest_directory' => '.', + 'preclean_directory' => 'install/images/alternatives', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_images_nonfree' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-images-nonfree', + 'source_subdir' => '/usr/lib*/drakx-installer-images', + 'dest_directory' => '.', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_stage2' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-stage2', + 'source_subdir' => '/usr/lib*/drakx-installer-stage2', + 'dest_directory' => '.', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_advertising' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-advertising', + 'source_subdir' => '/usr/share/drakx-installer-advertising', + 'dest_directory' => '.', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_installer_rescue' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'drakx-installer-rescue', + 'source_subdir' => '/usr/lib*/drakx-installer-rescue', + 'dest_directory' => 'install/stage2', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_release_notes' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'mageia-release-common', + 'source_subdir' => '/usr/share/doc/mageia-release-common', + 'grep_files' => 'release-notes.*', + 'dest_directory' => '.', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_syslinux' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'syslinux', + 'source_subdir' => '/usr/lib/syslinux/', + 'grep_files' => '\\(hdt\\|ifcpu\\|ldlinux\\|libcom32\\|libgpl\\|libmenu\\|libutil\\).c32', + 'dest_directory' => 'isolinux', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'unpack_pci_usb_ids' => { + 'class' => 'Youri::Submit::Action::Unpack', + 'options' => { + 'name' => 'ldetect-lst', + 'source_subdir' => '/usr/share/', + 'grep_files' => '\\(pci\\|usb\\).ids', + 'dest_directory' => 'isolinux', + 'unpack_inside_distribution_root' => '1', + }, + }, + 'mail' => { + 'class' => 'Youri::Submit::Action::Mail', + 'options' => { + 'mta' => '/usr/sbin/sendmail', + }, + }, + 'maintdb' => { + 'class' => 'Youri::Submit::Action::UpdateMaintDb', + }, + 'rebuild' => { + 'class' => 'Youri::Submit::Action::RebuildPackage', + 'options' => { + 'rules' => { + 'drakx-installer-binaries' => ['drakx-installer-images'], + 'drakx-kbd-mouse-x11' => ['drakx-installer-stage2'], + 'drakx-net' => ['drakx-installer-stage2'], + 'kernel-desktop-latest' => ['drakx-installer-images', 'kmod-virtualbox', 'kmod-xtables-addons'], + 'kernel-desktop586-latest' => ['drakx-installer-images', 'kmod-virtualbox', 'kmod-xtables-addons'], + 'kernel-server-latest' => ['kmod-virtualbox', 'kmod-xtables-addons'], + 'ldetect-lst' => ['drakx-installer-stage2'], + 'meta-task' => ['drakx-installer-stage2'], + 'perl' => ['drakx-installer-stage2'], + 'perl-URPM' => ['drakx-installer-stage2'], + 'rpm' => ['drakx-installer-stage2'], + 'rpm-mageia-setup' => ['drakx-installer-stage2'], + 'urpmi' => ['drakx-installer-stage2'], + }, + }, + }, + }, + 'posts' => { + 'genhdlist2' => { + 'class' => 'Youri::Submit::Post::Genhdlist2', + 'options' => { + 'command' => '/usr/bin/genhdlist2 --xml-info-filter ".lzma:xz -T4" --synthesis-filter ".cz:xz -7 -T8"', + }, + }, + 'genhdlist2_zstd' => { + 'class' => 'Youri::Submit::Post::Genhdlist2', + 'options' => { + 'command' => '/usr/bin/genhdlist2 --xml-info-filter ".lzma:xz -T4" --synthesis-filter ".cz:zstd -19 -T8"', + }, + }, + 'createrepo_mga6' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10', + }, + }, + 'createrepo_mga7' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10 --zck --zck-dict-dir /usr/share/mageia-repo-zdicts/mga7/', + }, + }, + 'createrepo_mga8' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10 --zck --zck-dict-dir /usr/share/mageia-repo-zdicts/mga7/', + }, + }, + 'createrepo_mga9' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10 --zck --zck-dict-dir /usr/share/mageia-repo-zdicts/mga7/', + }, + }, + 'createrepo_cauldron' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => 'createrepo_c --no-database --update --workers=10', + }, + }, + 'appstream_mga6' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-6-modifyrepo.sh', + }, + }, + 'appstream_mga7' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-7-modifyrepo.sh', + }, + }, + 'appstream_mga8' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-8-modifyrepo.sh', + }, + }, + 'appstream_mga9' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-9-modifyrepo.sh', + }, + }, + 'appstream_cauldron' => { + 'class' => 'Youri::Submit::Post::RunOnModifiedMedia', + 'options' => { + 'command' => '/distrib/appstream/appstream-cauldron-modifyrepo.sh', + }, + }, + 'clean_rpmsrate' => { + 'class' => 'Youri::Submit::Post::CleanRpmsrate', + }, + 'mirror' => { + 'class' => 'Youri::Submit::Post::Mirror', + 'options' => { + 'destination' => $buildsystem::var::repository::mirror_reporoot, + }, + }, + }, + }, + 'todo' => { + 'repository' => { + 'class' => 'Youri::Repository::Mageia', + 'options' => { + 'install_root' => $buildsystem::var::repository::bootstrap_reporoot, + 'upload_root' => '$home/uploads/', + 'upload_state' => 'todo done queue', + 'queue' => 'todo', + 'noarch' => 'x86_64', + 'svn' => "${buildsystem::var::mgarepo::svn_root_packages_ssh}/${buildsystem::var::distros::default_distro}", + }, + }, + 'checks' => { + 'tag' => $check_tag, + 'recency' => $check_recency, + 'queue_recency' => $check_queue_recency, + 'host' => $check_host, + 'rpmlint' => $check_rpmlint, + 'acl' => $check_acl, + 'source' => $check_source, + 'version' => $check_version, + 'deps' => { + 'class' => 'Youri::Submit::Check::Deps', + }, + }, + 'actions' => { + 'send' => { + 'class' => 'Youri::Submit::Action::Send', + 'options' => { + 'user' => $buildsystem::var::scheduler::login, + 'keep_svn_release' => 'yes', + 'uphost' => $buildsystem::var::scheduler::pkg_uphost, + 'root' => '$home/uploads', + 'ssh_key' => '$home/.ssh/id_rsa', + }, + }, + 'dependencies' => { + 'class' => 'Youri::Submit::Action::Dependencies', + 'options' => { + 'user' => $buildsystem::var::scheduler::login, + 'uphost' => $buildsystem::var::scheduler::pkg_uphost, + 'root' => '$home/uploads', + 'ssh_key' => '$home/.ssh/id_rsa', + }, + }, + 'rpminfo' => { + 'class' => 'Youri::Submit::Action::Rpminfo', + 'options' => { + 'user' => $buildsystem::var::scheduler::login, + 'uphost' => $buildsystem::var::scheduler::pkg_uphost, + 'root' => '$home/uploads', + 'ssh_key' => '$home/.ssh/id_rsa', + }, + }, + 'ulri' => { + 'class' => 'Youri::Submit::Action::Ulri', + 'options' => { + 'user' => $buildsystem::var::scheduler::login, + 'uphost' => $buildsystem::var::scheduler::pkg_uphost, + 'ssh_key' => '$home/.ssh/id_rsa', + }, + }, + }, + 'posts' => { + }, + }, + } +} diff --git a/modules/buildsystem/manifests/webstatus.pp b/modules/buildsystem/manifests/webstatus.pp new file mode 100644 index 00000000..49346dbc --- /dev/null +++ b/modules/buildsystem/manifests/webstatus.pp @@ -0,0 +1,44 @@ +class buildsystem::webstatus { + include buildsystem::var::webstatus + include buildsystem::var::scheduler + include apache::mod::php + + file { [ $buildsystem::var::webstatus::location, "${buildsystem::var::webstatus::location}/data" ]: + ensure => directory, + } + + $vhost = $buildsystem::var::webstatus::hostname + apache::vhost::base { $vhost: + aliases => { + '/uploads' => "${buildsystem::var::scheduler::homedir}/uploads", + '/autobuild/cauldron/x86_64/core/log/status.core.log' => "${buildsystem::var::webstatus::location}/autobuild/broken.php", + '/themes' => $buildsystem::var::webstatus::themes_dir, + }, + location => $buildsystem::var::webstatus::location, + content => template('buildsystem/vhost_webstatus.conf'), + } + + apache::vhost::base { "ssl_${vhost}": + vhost => $vhost, + use_ssl => true, + aliases => { + '/uploads' => "${buildsystem::var::scheduler::homedir}/uploads", + '/autobuild/cauldron/x86_64/core/log/status.core.log' => "${buildsystem::var::webstatus::location}/autobuild/broken.php", + '/themes' => $buildsystem::var::webstatus::themes_dir, + }, + location => $buildsystem::var::webstatus::location, + content => template('buildsystem/vhost_webstatus.conf'), + } + + git::snapshot { $buildsystem::var::webstatus::location: + source => $buildsystem::var::webstatus::git_url, + } + + file { '/etc/bs-webstatus.conf': + ensure => present, + content => template('buildsystem/bs-webstatus.conf'), + mode => '0644', + owner => root, + group => root, + } +} diff --git a/modules/buildsystem/manifests/youri_submit.pp b/modules/buildsystem/manifests/youri_submit.pp index e5e38fb8..6b4d7dc2 100644 --- a/modules/buildsystem/manifests/youri_submit.pp +++ b/modules/buildsystem/manifests/youri_submit.pp @@ -2,14 +2,10 @@ class buildsystem::youri_submit { include sudo include buildsystem::rpmlint include buildsystem::repository - include buildsystem::scheduler::var + include buildsystem::var::scheduler + include buildsystem::var::youri - $repository_root = $buildsystem::repository::dir - $sched_home_dir = $buildsystem::scheduler::var::homedir - $sched_login = $buildsystem::scheduler::var::login - $packages_archivedir = "$sched_home_dir/old" - - mga-common::local_script { + mga_common::local_script { 'mga-youri-submit': content => template('buildsystem/mga-youri-submit'); 'mga-youri-submit.wrapper': @@ -23,34 +19,54 @@ class buildsystem::youri_submit { sudo::sudoers_config { 'mga-youri-submit': content => template('buildsystem/sudoers.youri') } + $release_managers = group_members('mga-release_managers') # ordering is automatic : - # http://docs.puppetlabs.com/learning/ordering.html#autorequire + # https://docs.puppetlabs.com/learning/ordering.html#autorequire file { '/etc/youri/': ensure => 'directory'; - '/etc/youri/submit-todo.conf': - content => template('buildsystem/youri/submit-todo.conf'); - '/etc/youri/submit-upload.conf': - content => template('buildsystem/youri/submit-upload.conf'); '/etc/youri/acl.conf': content => template('buildsystem/youri/acl.conf'); + '/etc/youri/host.conf': + content => template('buildsystem/youri/host.conf'); + } + + buildsystem::youri_submit_conf{ 'upload': + tmpl_file => $buildsystem::var::youri::tmpl_youri_upload_conf, + } + buildsystem::youri_submit_conf{ 'todo': + tmpl_file => $buildsystem::var::youri::tmpl_youri_todo_conf, } - file { $packages_archivedir: + cron { 'Archive orphan packages from cauldron': + command => "/usr/local/bin/mga-clean-distrib --auto -v cauldron -d ${buildsystem::var::youri::packages_archivedir} -l ${buildsystem::var::scheduler::homedir}/tmp/upload", + hour => 5, + minute => 30, + user => $buildsystem::var::scheduler::login, + } + + file { $buildsystem::var::youri::packages_archivedir: ensure => 'directory', - owner => $sched_login, - require => File[$sched_home_dir], + owner => $buildsystem::var::scheduler::login, + require => File[$buildsystem::var::scheduler::homedir], } - tidy { $packages_archivedir: + tidy { $buildsystem::var::youri::packages_archivedir: type => 'ctime', recurse => true, age => '1w', matches => '*.rpm', } - # FIXME use /usr/local/ once it will be in @INC - file { '/usr/lib/perl5/vendor_perl/5.12.3/Youri/Repository/Mageia.pm': + include mga_common::var::perl + file { [ "${mga_common::var::perl::site_perl_dir}/Youri", + "${mga_common::var::perl::site_perl_dir}/Youri/Repository"]: + ensure => directory, + mode => '0755', + owner => root, + group => root, + } + file { "${mga_common::var::perl::site_perl_dir}/Youri/Repository/Mageia.pm": source => 'puppet:///modules/buildsystem/Mageia.pm', } diff --git a/modules/buildsystem/manifests/youri_submit_conf.pp b/modules/buildsystem/manifests/youri_submit_conf.pp new file mode 100644 index 00000000..28b911d9 --- /dev/null +++ b/modules/buildsystem/manifests/youri_submit_conf.pp @@ -0,0 +1,6 @@ +define buildsystem::youri_submit_conf($tmpl_file) { + $conf_name = $name + file { "/etc/youri/submit-${conf_name}.conf": + content => template($tmpl_file), + } +} |