[Mageia-dev] Distrib-coffee big failure of the century

Wed Oct 31 23:29:14 CET 2012

On 30/10/12 21:49, Barry Jackson wrote:
+> On 30/10/12 21:29, Johnny A. Solbu wrote:
+>> On Tuesday 30 October 2012 22:21, Barry Jackson wrote:
+>>> I will set --max-delete=500 locally in future.
+>>>
+>>> Any better ideas?
+>>
+>> I always run rsync manually twice. The first run I use «-n» to make
+>> sure my mirror isn't wiped, and then run rsync normally.
+>>
+>
+> I run it from a cron job via a script, so I could parse the output of a
+> dry run and count deletes for example, then let it decide whether to run
+> or not.
+> Thanks - worth investigating :)
+>
+I wrote the attached script. It checks several mirrors until it finds 
+one that has the normal number of files and does not fail for any other 
+reason.
+
+Currently I have distrib-coffee at the top of the possible mirror list, 
+but it is skipped because it only has a fraction of the files yet.
+
+It creates a couple of logs and runs as a cron job (or manually).
+
+If you feel inclined to use it please feel free to suggest any improvements.
+
+I certainly feel safer now.
+
+Barry
+-------------- next part --------------
+#!/bin/bash
+# ~/cronsync
+# Run with e.g. crontab:- 30 * * * *	$HOME/cronsync
+# Set using crontab -e
+###################
+# List of mirrors in preference order to use:-
+mymirrors=( \
+"rsync://distrib-coffee.ipsl.jussieu.fr:/pub/linux/Mageia/distrib" \
+"rsync://ftp.LinuxCabal.org/Mageia/distrib" \
+"rsync://ftp.acc.umu.se/mirror/mageia/distrib" \
+"rsync://mirrors.kernel.org:/mirrors/mageia/distrib" \
+"rsync://mageia.c3sl.ufpr.br/mageia/distrib" \
+)
+# Set limit for loss of # of files on server before it is skipped
+allowdropfiles=100
+###################
+# Personal config:-
+myexcludes="--exclude=debug --exclude=SRPMS --exclude=barjac"
+mydestination="/zmrepo/pub/linux/Mageia/"
+###################
+
+# Get minimum file count for mirror (set at allowdropfiles files less than the last actual file count)
+[[ -f .cronsync ]] || echo "140000" > $HOME/.cronsync
+minfiles=$(cat .cronsync)
+((actualfiles = $minfiles + $allowdropfiles))
+# Check a mirror
+chk_mirror()
+{
+echo "Checking $1 ..."
+chkout=$(rsync -rlptgoDhHSn \
+--stats \
+--delete-after \
+--delete \
+--delete-excluded \
+--protect-args \
+$myexcludes \
+"$1" \
+"$mydestination" | grep "Number of files:" )
+status1="$?"
+files=$(echo $chkout | cut -d' ' -f4)
+[[ $files -lt $minfiles ]] && echo "$(date +%d/%m/%Y-%H:%M): Only $files/$actualfiles files in $1 !" | tee -a $HOME/cronsync_error.log
+[[ $status1 > 0 ]] && echo "$(date +%d/%m/%Y-%H:%M): Error: $status1 while checking $1" | tee -a $HOME/cronsync_error.log
+([[ $status1 = 0 ]] && [[ $files -gt $minfiles ]]) || return 1
+((chkcount = $files - $allowdropfiles))
+echo $chkcount > $HOME/.cronsync
+return 0
+}
+
+# Loop through mirror list checking until one succeeds
+find_good_mirror()
+{
+for mirr in ${mymirrors[@]}; do
+chk_mirror $mirr
+[[ $? = 0 ]] && { echo "Found good mirror - $mirr"; return 0; }
+done
+return 1
+}
+
+#=======================Main Program starts here=====================
+
+# Check if rsync is already running (maybe last cron sync taking for ever?)
+ps aux | grep -q [r]sync
+if [[ $? > 0 ]]; then
+
+# Find a good mirror from the list (with files in it)
+find_good_mirror || { echo "$(date +%d/%m/%Y-%H:%M) Can't find a good mirror - aborting" | tee -a $HOME/cronsync_error.log; exit 1; }
+
+echo "Syncing from $mirr ..."
+
+# Live run with --max-delete set to 1000 just in case ;)
+rsync -rlptgoDhHS \
+--progress \
+--stats \
+--delete-after \
+--delete \
+--max-delete=1000 \
+--delete-excluded \
+--protect-args \
+$myexcludes \
+"$mirr" \
+"$mydestination" | tee $HOME/cronsync_last.log
+
+status=$?
+[[ $status = 0 ]] && { echo "Update complete"; exit 0; }
+[[ $status > 0 ]] && { echo "Live run error - $(date +%d/%m/%Y-%H:%M) = $status" >> $HOME/cronsync_error.log; exit 1; }
+else
+echo "rsync already running, skipping update - $(date +%d/%m/%Y-%H:%M)" >> $HOME/cronsync_error.log
+exit 1
+fi
+