From 6efe180a02aa2c41808b817df504781a434673d0 Mon Sep 17 00:00:00 2001 From: Colin Guthrie Date: Thu, 31 Jul 2014 00:30:24 +0100 Subject: move: Totally rewrite mga-move-pkg to handle multiple arguments. This allows us to specify multiple packages (from various distro releases and sections) to be processed. All the arguments are parsed into arrays and then these arrays are looped over and processed all at once. If the --sync option is given then it will only update the affected hdlists and md5sums once. The idea next is to write a wrapper in python which will take advisory bug numbers as arguments and then: 1. Ensure each has an ID assigned (this has to be done first) 2. Calculate all the packages involved. 3. Run this script with the --sync --no-confirm options 4. Close all the bugs with a comment + link to the Advisory (hence the need for the advisory ID in step 1 and the desire to use python as this is how the git->bz integration works). This scheme should save a lot of time, but stops just short of fully automating things. Sadly the cronjob that runs "mgaadv mksite" would likely have to be disabled, but we should be able to add a new status field that does these moves too and incorporate that into mgaadv and thus provide full automation \o/ --- mga-move-pkg | 327 +++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 196 insertions(+), 131 deletions(-) diff --git a/mga-move-pkg b/mga-move-pkg index 143d410..6662476 100755 --- a/mga-move-pkg +++ b/mga-move-pkg @@ -22,179 +22,244 @@ if [ "$1" = "--no-confirm" ]; then fi backport= -if [ "$1" = "--backports" ]; then - shift - backport=$1 +if [ "$1" = "--backport" ]; then + backport=y shift fi -if [ $# != 3 ]; then - echo "Usage: $0 [--dry-run] [--sync] [--no-confirm] [--backport ]
" >&2 +function usage() +{ + echo "Usage: $(basename $0) [--dry-run] [--sync] [--no-confirm] [--backport] /
/ [/
/...]" >&2 exit 1 -fi +} +[ $# -lt 1 ] && usage -valid_release "$1" -valid_section "$2" - -release="$1" -section="$2" -srcname="$3" - -exactsrpm= -sourcesubsection='updates_testing' -destinationsubsection='updates' -if [ 'cauldron' = "$release" ]; then - if [ -n "$backport" ]; then - echo "Error: Cannot use --backport with cauldron" >&2 - exit 1 - fi - destinationsubsection='release' -fi +declare subsection='updates'; +[ -n "$backport" ] && subsection='backports'; -if [ -n "$backport" ]; then - sourcesubsection='backports_testing' - destinationsubsection='backports' -fi +declare -a _releases; +declare -a _sections; +declare -a _srcsubsections; +declare -a _destsubsections; +declare -a _pkgs; +declare -a _pkgfiles; +declare -a _oldpkgfiles; -for file in "$distribdir/$release/SRPMS/$section/$sourcesubsection/"*.rpm; do - pkg=$(rpm -qp --qf '%{NAME}' "$file") +# Assume the input is the full srpm name (used to avoid validation) +declare _fullsrpmname="y" - # Check exact match - bfile=$(basename "$file") - if [ "$bfile" = "$srcname" ]; then - exactsrpm=y - srcname=$pkg - srcpkg=$bfile - break - fi +function parse_args() +{ + local -a tmptriplet; + local -i i=0; - # Check for src package name - if [ "$pkg" = "$srcname" ]; then - srcpkg=$bfile - break - fi -done + for arg in "$@"; do + OIFS="$IFS"; IFS=/; tmptriplet=($arg); IFS="$OIFS"; -if [ -z "$srcpkg" ]; then - echo "The package '$srcname' could not be found in the '$release/$section/$sourcesubsection' repository." >&2 - exit 2 -fi + [ ${#tmptriplet[*]} -ne 3 ] && usage + + _releases[$i]=${tmptriplet[0]} + _sections[$i]=${tmptriplet[1]} + _pkgs[$i]=${tmptriplet[2]} -oldsrcpkg= -if [ 'release' = "$destinationsubsection" ]; then - # In cauldron we also have to move the old version from release to ~schedbot/old/ - # We know srcname is a %{name} now. - for file in "$distribdir/$release/SRPMS/$section/release/"*.rpm; do - pkg=$(rpm -qp --qf '%{NAME}' "$file") - if [ "$pkg" = "$srcname" ]; then - oldsrcpkg=$bfile - break + valid_release ${_releases[$i]} + valid_section ${_sections[$i]} + + _srcsubsections[$i]="${subsection}_testing" + _destsubsections[$i]=$subsection + + # Special case in cauldron where we move updates_testing -> release + if [ 'cauldron' = "${_releases[$i]}" ]; then + if [ -n "$backport" ]; then + echo "Error: Cannot use --backport with cauldron" >&2 + exit 1 + fi + _destsubsections[$i]="release" fi + ((++i)) done +} +parse_args "$@" - # (cg) Is this a valid check? What if it's a new package introduced first into testing - # Ultimately this is not very likely, so ignore it for now. - if [ -z "$oldsrcpkg" ]; then - echo "The package '$srcname' could not be found in the '$release/$section/$sourcesubsection' repository." >&2 - exit 2 - fi -fi +function find_srpms() +{ + local file + local pkg + local bfile + local exactsrpm + for ((i=0;i<${#_releases[*]};++i)); do + exactsrpm= + for file in "$distribdir/${_releases[$i]}/SRPMS/${_sections[$i]}/${_srcsubsections[$i]}/"*.rpm; do + pkg=$(rpm -qp --qf '%{NAME}' "$file") + + # Check exact match + bfile=$(basename "$file") + if [ "$bfile" = "${_pkgs[$i]}" ]; then + exactsrpm=y + _pkgs[$i]=$pkg + _pkgfiles[$i]=$bfile + break + fi + + # Check for src package name + if [ "$pkg" = "${_pkgs[$i]}" ]; then + _pkgfiles[$i]=$bfile + break + fi + done + + # If we didn't find an exact srpm, then update our global var accordingly + [ -z "$exactsrpm" ] && _fullsrpmname= + + if [ -z "${_pkgfiles[$i]}" ]; then + echo "The package '${_pkgs[$i]}' could not be found in the '${_releases[$i]}/${_sections[$i]}/${_srcsubsections[$i]}' repository." >&2 + exit 2 + fi + + _oldpkgfiles[$i]= + if [ 'release' = "${_destsubsections[$i]}" ]; then + # In cauldron we also have to move the old version from release to ~schedbot/old/ + # We know srcname is a %{name} now. + for file in "$distribdir/${_releases[$i]}/SRPMS/${_sections[$i]}/release/"*.rpm; do + pkg=$(rpm -qp --qf '%{NAME}' "$file") + if [ "$pkg" = "${_pkgs[$i]}" ]; then + _oldpkgfiles[$i]=$bfile + break + fi + done + + # (cg) Is this a valid check? What if it's a new package introduced first into testing + # Ultimately this is not very likely, so ignore it for now. + if [ -z "${_oldpkgfiles[$i]}" ]; then + echo "The package '${_pkgs[$i]}' could not be found in the '${_releases[$i]}/${_sections[$i]}/${_srcsubsections[$i]}' repository." >&2 + exit 2 + fi + fi + done +} +find_srpms # Safety check -if [ -z "$exactsrpm" -o -z "$noconfirm" ]; then - echo "This SRPM (and matching binaries) will be moved from '$sourcesubsection' to '$destinationsubsection':" - echo "- $srcpkg" - echo - if [ 'release' = "$destinationsubsection" ]; then - echo "This SRPM (and matching binaries) will be moved from 'release' to '\~schedbot/old':" - echo "- $oldsrcpkg" +function sanity_check() +{ + local cauldronmsg + + if [ "$_fullsrpmname" != "y" -o -z "$noconfirm" ]; then + echo "The following SRPMs (and matching binaries) will be moved:" + for ((i=0;i<${#_releases[*]};++i)); do + echo "- ${_pkgfiles[$i]}" + if [ 'release' = "${_destsubsections[$i]}" -a -n "${_oldpkgfiles[$i]}" ]; then + cauldronmsg="${cauldronmsg}- ${_oldpkgfiles[$i]}\n" + fi + done echo + if [ -n "$cauldronmsg" ]; then + echo "The following SRPMs (and matching binaries) will be moved from 'release' to '\~schedbot/old':" + echo "$cauldronmsg" + fi + + read -n 1 -p "Are you sure? [Y/n] " yn + [ -n "$yn" ] && echo + [ -z "$yn" -o "Y" = "$yn" ] && yn=y + [ "y" != "$yn" ] && exit fi + return 0 +} +sanity_check - read -n 1 -p "Are you sure? [Y/n] " yn - [ -n "$yn" ] && echo - [ -z "$yn" -o "Y" = "$yn" ] && yn=y - [ "y" != "$yn" ] && exit -fi +declare _mailcontent=$(mktemp) +declare _mailcommands=$(mktemp) function movepkg() { - srcdir="$1" - destdir="$2" - srcpkg="$3" - content="$4" - commands="$5" + local srcdir="$1" + local destdir="$2" + local srcpkg="$3" + local file + local srpm for file in "$srcdir/"*.rpm; do srpm=$(rpm -qp --qf '%{SOURCERPM}' "$file") if [ "$srpm" = "$srcpkg" ]; then - echo " $(basename $file)" | tee -a "$content" - $dryrun mv -v -f "$file" "$destdir" >>"$commands" 2>&1 + echo " $(basename $file)" | tee -a "$_mailcontent" + $dryrun mv -v -f "$file" "$destdir" >>"$_mailcommands" 2>&1 fi done } -mailcontent=$(mktemp) -mailcommands=$(mktemp) +function movepkgs() +{ + local arch -if [ 'release' = "$destinationsubsection" ]; then - echo "" - echo "Moving binary and source rpms:" | tee -a "$mailcontent" - for arch in $arches; do - echo "- $arch:" | tee -a "$mailcontent" - movepkg "$distribdir/$release/$arch/media/$section/release" "/var/lib/schedbot/old/" "$oldsrcpkg" "$mailcontent" "$mailcommands" - movepkg "$distribdir/$release/$arch/media/debug/$section/release" "/var/lib/schedbot/old/" "$oldsrcpkg" "$mailcontent" "$mailcommands" - done - echo "- source:" | tee -a "$mailcontent" - echo " $oldsrcpkg" | tee -a "$mailcontent" - echo >> "$mailcontent" - echo >> "$mailcontent" - $dryrun mv -v -f "$distribdir/$release/SRPMS/$section/release/$oldsrcpkg" "/var/lib/schedbot/old/" >>"$mailcommands" 2>&1 -fi + for ((i=0;i<${#_releases[*]};++i)); do + if [ 'release' = "${_destsubsections[$i]}" -a -n "${_oldpkgfiles[$i]}" ]; then + echo "" + echo "Moving binary and source rpms:" | tee -a "$_mailcontent" + for arch in $arches; do + echo "- $arch:" | tee -a "$_mailcontent" + movepkg "$distribdir/${_releases[$i]}/$arch/media/${_sections[$i]}/release" "/var/lib/schedbot/old/" "${_oldpkgfiles[$i]}" + movepkg "$distribdir/${_releases[$i]}/$arch/media/debug/${_sections[$i]}/release" "/var/lib/schedbot/old/" "${_oldpkgfiles[$i]}" + done + echo "- source:" | tee -a "$_mailcontent" + echo " ${_oldpkgfiles[$i]}" | tee -a "$_mailcontent" + echo >> "$_mailcontent" + echo >> "$_mailcontent" + $dryrun mv -v -f "$distribdir/${_releases[$i]}/SRPMS/${_sections[$i]}/release/${_oldpkgfiles[$i]}" "/var/lib/schedbot/old/" >>"$_mailcommands" 2>&1 + fi -echo "" -echo "Moving binary and source rpms:" | tee -a "$mailcontent" -for arch in $arches; do - echo "- $arch:" | tee -a "$mailcontent" - movepkg "$distribdir/$release/$arch/media/$section/$sourcesubsection" "$distribdir/$release/$arch/media/$section/$destinationsubsection/" "$srcpkg" "$mailcontent" "$mailcommands" - movepkg "$distribdir/$release/$arch/media/debug/$section/$sourcesubsection" "$distribdir/$release/$arch/media/debug/$section/$destinationsubsection/" "$srcpkg" "$mailcontent" "$mailcommands" -done -echo "- source:" | tee -a "$mailcontent" -echo " $srcpkg" | tee -a "$mailcontent" -echo >> "$mailcontent" -echo >> "$mailcontent" -$dryrun mv -v -f "$distribdir/$release/SRPMS/$section/$sourcesubsection/$srcpkg" "$distribdir/$release/SRPMS/$section/$destinationsubsection/" >>"$mailcommands" 2>&1 + echo "" + echo "Moving binary and source rpms:" | tee -a "$_mailcontent" + for arch in $arches; do + echo "- $arch:" | tee -a "$_mailcontent" + movepkg "$distribdir/${_releases[$i]}/$arch/media/${_sections[$i]}/${_srcsubsections[$i]}" "$distribdir/${_releases[$i]}/$arch/media/${_sections[$i]}/${_destsubsections[$i]}/" "${_pkgfiles[$i]}" "$_mailcontent" "$_mailcommands" + movepkg "$distribdir/${_releases[$i]}/$arch/media/debug/${_sections[$i]}/${_srcsubsections[$i]}" "$distribdir/${_releases[$i]}/$arch/media/debug/${_sections[$i]}/${_destsubsections[$i]}/" "${_pkgfiles[$i]}" "$_mailcontent" "$_mailcommands" + done + echo "- source:" | tee -a "$_mailcontent" + echo " ${_pkgfiles[$i]}" | tee -a "$_mailcontent" + echo >> "$_mailcontent" + echo >> "$_mailcontent" + $dryrun mv -v -f "$distribdir/${_releases[$i]}/SRPMS/${_sections[$i]}/${_srcsubsections[$i]}/${_pkgfiles[$i]}" "$distribdir/${_releases[$i]}/SRPMS/${_sections[$i]}/${_destsubsections[$i]}/" >>"$_mailcommands" 2>&1 + done +} +movepkgs if [ -n "$dryrun" ]; then echo echo "Raw Commands:" - cat "$mailcommands" + cat "$_mailcommands" else if [ -z "$sync" ]; then - echo >>"$mailcontent" - echo "Warning: hdlist update+sync was not performed during this move" >>"$mailcontent" - echo >>"$mailcontent" - echo >>"$mailcontent" + echo >>"$_mailcontent" + echo "Warning: hdlist update+sync was not performed during this move" >>"$_mailcontent" + echo >>"$_mailcontent" + echo >>"$_mailcontent" fi - if [ -n "$backport" ]; then - export SIG="https://bugs.mageia.org/show_bug.cgi?id=$backport" - $dryrun mutt -e 'set from="Mageia Backports "' \ - -e 'set signature="echo $SIG|"' \ - -e 'set sig_dashes=no' \ - backports-announce@ml.mageia.org -s "Package Backport: MGA $release/$section: $srcname" \ - -i "$mailcontent" - fi - - cat "$mailcommands" >>"$mailcontent" - /usr/bin/mailx -s "$0 $release $section $srcname" -S "from=$moveupdate_mailfrom" "$moveupdate_mailto" < "$mailcontent" + cat "$_mailcommands" >>"$_mailcontent" + /usr/bin/mailx -s "$(basename $0) ${#_releases[*]} source package(s) moved" -S "from=$moveupdate_mailfrom" "$moveupdate_mailto" < "$_mailcontent" if [ -n "$sync" ]; then - update_distro_hdlist "$release" "$section" "$sourcesubsection" - update_distro_hdlist "$release" "$section" "$destinationsubsection" - update_common_MD5SUM "$release" - mirror_repository "$release" + updated="xxxx" + for ((i=0;i<${#_releases[*]};++i)); do + if ! inlist "${_releases[$i]}/${_sections[$i]}/${_srcsubsections[$i]}" "$updated"; then + update_distro_hdlist "${_releases[$i]}" "${_sections[$i]}" "${_srcsubsections[$i]}" + updated="$updated ${_releases[$i]}/${_sections[$i]}/${_srcsubsections[$i]}" + fi + if ! inlist "${_releases[$i]}/${_sections[$i]}/${_destsubsections[$i]}" "$updated"; then + update_distro_hdlist "${_releases[$i]}" "${_sections[$i]}" "${_destsubsections[$i]}" + updated="$updated ${_releases[$i]}/${_sections[$i]}/${_destsubsections[$i]}" + fi + done + + updated="xxxx" + for ((i=0;i<${#_releases[*]};++i)); do + if ! inlist "${_releases[$i]}" "$updated"; then + update_common_MD5SUM "${_release[$i]}" + mirror_repository "${_release[$i]}" + updated="$updated ${_release[$i]}" + fi + done fi fi -rm -f "$mailcontent" "$mailcommands" +rm -f "$_mailcontent" "$_mailcommands" -- cgit v1.2.1