from BuildManager.fileutil import * from BuildManager.package import * from BuildManager import * import _thread import subprocess import sys, os import time import shutil import glob import codecs __all__ = ["PackageBuilder"] GLOBAL_PKGLIST_LOCK = _thread.allocate_lock() STAGE_UNPACK = 0 STAGE_PREP = 1 STAGE_GENERATE_BUILDREQUIRES = 2 STAGE_COMPILE = 3 STAGE_INSTALL = 4 STAGE_SOURCE = 5 STAGE_BINARY = 6 STAGE_ALL = 7 STAGE_DICT = {"unpack": STAGE_UNPACK, "prep": STAGE_PREP, "generate_buildrequires": STAGE_GENERATE_BUILDREQUIRES, "compile": STAGE_COMPILE, "install": STAGE_INSTALL, "source": STAGE_SOURCE, "binary": STAGE_BINARY, "all": STAGE_ALL} class PackageBuilder: def __init__(self, opts): self.opts = opts self.stage = STAGE_DICT[opts.mode] def run(self): self.pkglist = PackageList() logger.info("creating package list") for filename in self.opts.args: pkg = Package(filename, self.opts.build_log, self.opts.forceunpack) if pkg.type == 'rpm': logger.info("skipping %s, not a src.rpm" % filename) break for ignore in self.opts.ignore: if ignore.match(pkg.name): break else: self.pkglist.append(pkg) for dir in self.opts.filter_renew: filterpkglist(pkglist, dir, "not_has_ge") for dir in self.opts.filter_refresh: filterpkglist(pkglist, dir, "not_has_lt") self.pkgsleft = len(self.pkglist) if self.pkgsleft == 0: logger.info("no packages to process") return True elif self.pkgsleft > 1 and self.pkgsleft % 10 != 0: logger.info("package list has %d packages" % pkgsleft) self.pkglist_lock = _thread.allocate_lock() self.failures = 0 if self.opts.parallel != 1: logger.info("starting threads") for i in range(self.opts.parallel-1): _thread.start_new_thread(self_processlist, ()) self._processlist() while self.pkgsleft > 0: time.sleep(2) return not self.failures def _processlist(self): while 1: self.pkglist_lock.acquire() if not self.pkglist: self.pkglist_lock.release() return 1 if self.pkgsleft % 10 == 0: logger.info("package list has %d packages" % self.pkgsleft) pkg = self.pkglist.pop(0) logger.info("processing package %s-%s-%s" % (pkg.name, pkg.version, pkg.release)) self.pkglist_lock.release() ret = buildpkg(pkg=pkg, stage=self.stage, unpack_dir=self.opts.unpack_dir, passtrough=" ".join(self.opts.options), show_log=self.opts.show_log, dryrun=self.opts.dryrun) if ret: if pkg.type == "srpm": if self.opts.move_succeeded_srpm: move_file(pkg.file, self.opts.move_succeeded_srpm, dryrun=self.opts.dryrun) elif self.opts.copy_succeeded_srpm: copy_file(pkg.file, self.opts.copy_succeeded_srpm, dryrun=self.opts.dryrun) elif self.opts.remove_succeeded_srpm: logger.info("removing %s" % pkg.file) if not self.opts.dryrun: os.unlink(pkg.file) if self.opts.move_srpm: dir = os.path.join(pkg.builddir, "SRPMS") for file in glob.glob(os.path.join(dir, "*.src.rpm")): move_file(os.path.join(dir, file), self.opts.move_srpm, dryrun=self.opts.dryrun) if self.opts.move_rpm: dir = os.path.join(pkg.builddir, "RPMS") for subdir in os.listdir(dir): subdir = os.path.join(dir, subdir) for file in os.listdir(subdir): move_file(os.path.join(subdir, file), self.opts.move_rpm, dryrun=self.opts.dryrun) if self.opts.move_log: move_file(pkg.log, self.opts.move_log, dryrun=self.opts.dryrun) if self.opts.clean_topdir or self.opts.clean_on_success: if pkg.builddir != "/": logger.info("cleaning build directory") if not self.opts.dryrun: shutil.rmtree(pkg.builddir) else: self.failures += 1 if pkg.type == "srpm": if self.opts.move_failed_srpm: move_file(pkg.file, self.opts.move_failed_srpm, dryrun=self.opts.dryrun) elif self.opts.copy_failed_srpm: copy_file(pkg.file, self.opts.copy_failed_srpm, dryrun=self.opts.dryrun) elif self.opts.remove_failed_srpm: logger.info("removing %s" % pkg.file) if not self.opts.dryrun: os.unlink(pkg.file) if self.opts.move_failed_log: move_file(pkg.log, self.opts.move_failed_log, dryrun=self.opts.dryrun) if self.opts.clean_topdir: if pkg.builddir != "/": logger.info("cleaning build directory") if not self.opts.dryrun: shutil.rmtree(pkg.builddir) self.pkglist_lock.acquire() self.pkgsleft -= 1 self.pkglist_lock.release() def filterpkglist(pkglist, directory, rule): filterlist = PackageList() logger.info("creating package list filter for "+directory) for filename in os.listdir(directory): filename = os.path.join(directory, filename) if os.path.isfile(filename): filterlist.append(Package(filename)) logger.info("filtering") if rule[:4] == "not_": filterfunc_tmp = getattr(filterlist, rule[4:]) filterfunc = lambda x: not filterfunc_tmp(x) else: filterfunc = getattr(filterlist, rule) pkglist[:] = list(filter(filterfunc, pkglist)) def buildpkglist(pkglist, stage, unpack_dir, passtrough="", show_log=0, dryrun=0): while 1: GLOBAL_PKGLIST_LOCK.acquire() if not pkglist: GLOBAL_PKGLIST_LOCK.release() return 1 pkg = pkglist[0] del pkglist[0] GLOBAL_PKGLIST_LOCK.release() buildpkg(pkg, stage, unpack_dir, passtrough, show_log, dryrun) def buildpkg(pkg, stage, unpack_dir, passtrough="", show_log=0, dryrun=0): stagestr = ["unpacking", "running prep stage", "running prep and generate_buildrequires stages", "running prep and compile stage", "running prep, compile, and install stages", "building source package", "building binary packages", "building source and binary packages"][stage] logger.info(stagestr) ret = 0 if pkg.type == "srpm" and not (dryrun or pkg.unpack(unpack_dir)): logger.error("failed unpacking") return 0 else: status = 0 if stage != STAGE_UNPACK: stagechar = ["p","r","c","i","s","b","a"][stage-1] tmppath = "" nodeps = "" noclean = "--noclean" topdir = "--define '_topdir %s'" % pkg.builddir if not dryrun and os.path.isdir(pkg.builddir+"/BUILDROOT"): tmppath = "--define '_tmppath %s/BUILDROOT'" % pkg.builddir if stage == STAGE_PREP: nodeps = "--nodeps" if '--clean' in passtrough.split(): noclean = "" cmd = "rpmbuild -b%s %s %s %s %s %s %s 2>&1" % \ (stagechar, nodeps, noclean, topdir, tmppath, passtrough, pkg.spec) logger.debug("rpmbuild command: "+cmd) if not dryrun: log = codecs.open(pkg.log, "w", 'utf-8', errors = "replace") pop = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) of = pop.stdout.fileno() while True: status = pop.poll() if status is not None: break try: data = os.read(of, 8192).decode(sys.getfilesystemencoding()) log.write(data) log.flush() if show_log: sys.stdout.write(data) except UnicodeDecodeError: pass log.close() if status == 0: logger.info("succeeded!") ret = 1 else: logger.error("failed!") ret = 0 return ret # vim:ts=4:sw=4