blob: a6624a97239efd659a7bb15c3caaecd06a806b65 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">
<HTML>
<HEAD>
<TITLE> [Mageia-dev] Distrib-coffee big failure of the century
</TITLE>
<LINK REL="Index" HREF="index.html" >
<LINK REL="made" HREF="mailto:mageia-dev%40mageia.org?Subject=Re%3A%20%5BMageia-dev%5D%20Distrib-coffee%20big%20failure%20of%20the%20century&In-Reply-To=%3C5091A63A.70403%40zen.co.uk%3E">
<META NAME="robots" CONTENT="index,nofollow">
<META http-equiv="Content-Type" content="text/html; charset=us-ascii">
<LINK REL="Previous" HREF="019673.html">
<LINK REL="Next" HREF="019713.html">
</HEAD>
<BODY BGCOLOR="#ffffff">
<H1>[Mageia-dev] Distrib-coffee big failure of the century</H1>
<B>Barry Jackson</B>
<A HREF="mailto:mageia-dev%40mageia.org?Subject=Re%3A%20%5BMageia-dev%5D%20Distrib-coffee%20big%20failure%20of%20the%20century&In-Reply-To=%3C5091A63A.70403%40zen.co.uk%3E"
TITLE="[Mageia-dev] Distrib-coffee big failure of the century">zen25000 at zen.co.uk
</A><BR>
<I>Wed Oct 31 23:29:14 CET 2012</I>
<P><UL>
<LI>Previous message: <A HREF="019673.html">[Mageia-dev] Distrib-coffee big failure of the century
</A></li>
<LI>Next message: <A HREF="019713.html">[Mageia-dev] Distrib-coffee big failure of the century
</A></li>
<LI> <B>Messages sorted by:</B>
<a href="date.html#19712">[ date ]</a>
<a href="thread.html#19712">[ thread ]</a>
<a href="subject.html#19712">[ subject ]</a>
<a href="author.html#19712">[ author ]</a>
</LI>
</UL>
<HR>
<!--beginarticle-->
<PRE>On 30/10/12 21:49, Barry Jackson wrote:
><i> On 30/10/12 21:29, Johnny A. Solbu wrote:
</I>>><i> On Tuesday 30 October 2012 22:21, Barry Jackson wrote:
</I>>>><i> I will set --max-delete=500 locally in future.
</I>>>><i>
</I>>>><i> Any better ideas?
</I>>><i>
</I>>><i> I always run rsync manually twice. The first run I use «-n» to make
</I>>><i> sure my mirror isn't wiped, and then run rsync normally.
</I>>><i>
</I>><i>
</I>><i> I run it from a cron job via a script, so I could parse the output of a
</I>><i> dry run and count deletes for example, then let it decide whether to run
</I>><i> or not.
</I>><i> Thanks - worth investigating :)
</I>><i>
</I>I wrote the attached script. It checks several mirrors until it finds
one that has the normal number of files and does not fail for any other
reason.
Currently I have distrib-coffee at the top of the possible mirror list,
but it is skipped because it only has a fraction of the files yet.
It creates a couple of logs and runs as a cron job (or manually).
If you feel inclined to use it please feel free to suggest any improvements.
I certainly feel safer now.
Barry
-------------- next part --------------
#!/bin/bash
# ~/cronsync
# Run with e.g. crontab:- 30 * * * * $HOME/cronsync
# Set using crontab -e
###################
# List of mirrors in preference order to use:-
mymirrors=( \
"<A HREF="rsync://distrib-coffee.ipsl.jussieu.fr:/pub/linux/Mageia/distrib">rsync://distrib-coffee.ipsl.jussieu.fr:/pub/linux/Mageia/distrib</A>" \
"<A HREF="rsync://ftp.LinuxCabal.org/Mageia/distrib">rsync://ftp.LinuxCabal.org/Mageia/distrib</A>" \
"<A HREF="rsync://ftp.acc.umu.se/mirror/mageia/distrib">rsync://ftp.acc.umu.se/mirror/mageia/distrib</A>" \
"<A HREF="rsync://mirrors.kernel.org:/mirrors/mageia/distrib">rsync://mirrors.kernel.org:/mirrors/mageia/distrib</A>" \
"<A HREF="rsync://mageia.c3sl.ufpr.br/mageia/distrib">rsync://mageia.c3sl.ufpr.br/mageia/distrib</A>" \
)
# Set limit for loss of # of files on server before it is skipped
allowdropfiles=100
###################
# Personal config:-
myexcludes="--exclude=debug --exclude=SRPMS --exclude=barjac"
mydestination="/zmrepo/pub/linux/Mageia/"
###################
# Get minimum file count for mirror (set at allowdropfiles files less than the last actual file count)
[[ -f .cronsync ]] || echo "140000" > $HOME/.cronsync
minfiles=$(cat .cronsync)
((actualfiles = $minfiles + $allowdropfiles))
# Check a mirror
chk_mirror()
{
echo "Checking $1 ..."
chkout=$(rsync -rlptgoDhHSn \
--stats \
--delete-after \
--delete \
--delete-excluded \
--protect-args \
$myexcludes \
"$1" \
"$mydestination" | grep "Number of files:" )
status1="$?"
files=$(echo $chkout | cut -d' ' -f4)
[[ $files -lt $minfiles ]] && echo "$(date +%d/%m/%Y-%H:%M): Only $files/$actualfiles files in $1 !" | tee -a $HOME/cronsync_error.log
[[ $status1 > 0 ]] && echo "$(date +%d/%m/%Y-%H:%M): Error: $status1 while checking $1" | tee -a $HOME/cronsync_error.log
([[ $status1 = 0 ]] && [[ $files -gt $minfiles ]]) || return 1
((chkcount = $files - $allowdropfiles))
echo $chkcount > $HOME/.cronsync
return 0
}
# Loop through mirror list checking until one succeeds
find_good_mirror()
{
for mirr in ${mymirrors[@]}; do
chk_mirror $mirr
[[ $? = 0 ]] && { echo "Found good mirror - $mirr"; return 0; }
done
return 1
}
#=======================Main Program starts here=====================
# Check if rsync is already running (maybe last cron sync taking for ever?)
ps aux | grep -q [r]sync
if [[ $? > 0 ]]; then
# Find a good mirror from the list (with files in it)
find_good_mirror || { echo "$(date +%d/%m/%Y-%H:%M) Can't find a good mirror - aborting" | tee -a $HOME/cronsync_error.log; exit 1; }
echo "Syncing from $mirr ..."
# Live run with --max-delete set to 1000 just in case ;)
rsync -rlptgoDhHS \
--progress \
--stats \
--delete-after \
--delete \
--max-delete=1000 \
--delete-excluded \
--protect-args \
$myexcludes \
"$mirr" \
"$mydestination" | tee $HOME/cronsync_last.log
status=$?
[[ $status = 0 ]] && { echo "Update complete"; exit 0; }
[[ $status > 0 ]] && { echo "Live run error - $(date +%d/%m/%Y-%H:%M) = $status" >> $HOME/cronsync_error.log; exit 1; }
else
echo "rsync already running, skipping update - $(date +%d/%m/%Y-%H:%M)" >> $HOME/cronsync_error.log
exit 1
fi
</PRE>
<!--endarticle-->
<HR>
<P><UL>
<!--threads-->
<LI>Previous message: <A HREF="019673.html">[Mageia-dev] Distrib-coffee big failure of the century
</A></li>
<LI>Next message: <A HREF="019713.html">[Mageia-dev] Distrib-coffee big failure of the century
</A></li>
<LI> <B>Messages sorted by:</B>
<a href="date.html#19712">[ date ]</a>
<a href="thread.html#19712">[ thread ]</a>
<a href="subject.html#19712">[ subject ]</a>
<a href="author.html#19712">[ author ]</a>
</LI>
</UL>
<hr>
<a href="https://www.mageia.org/mailman/listinfo/mageia-dev">More information about the Mageia-dev
mailing list</a><br>
</body></html>
|
