aboutsummaryrefslogtreecommitdiffstats
path: root/lib/Downloads.php
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Downloads.php')
-rw-r--r--lib/Downloads.php295
1 files changed, 194 insertions, 101 deletions
diff --git a/lib/Downloads.php b/lib/Downloads.php
index f4b21f85f..69a0e38ad 100644
--- a/lib/Downloads.php
+++ b/lib/Downloads.php
@@ -88,8 +88,8 @@ class Downloads
/**
* Sort 2D array by multiple associative or numeric keys.
* $sorted_array = self::sort_2d_array_by_multiple_keys($unsorted_array, 'first key', 'second', ...);
- *
- * based on SortArray http://php.net/manual/en/function.usort.php#42535
+ *
+ * Note that keys are preserved!
*
* @param array $unsorted_array
*
@@ -103,21 +103,27 @@ class Downloads
*/
public static function sort_2d_array_by_multiple_keys()
{
- $arguments = func_get_args();
- $array = $arguments[0];
- $anonymous_function = '';
- $num_of_arguments = count($arguments);
- for ($cur_argument = 1; $cur_argument < $num_of_arguments; $cur_argument++) {
- $anonymous_function .= "if (\$first['$arguments[$cur_argument]'] != \$second['$arguments[$cur_argument]']) {";
- $anonymous_function .= " \$compare_result = strcoll(\$first['$arguments[$cur_argument]'], \$second['$arguments[$cur_argument]']);";
- $anonymous_function .= " if (0 == \$compare_result) { return 0; };";
- $anonymous_function .= " return ((0 > \$compare_result) ? -1 : 1);";
- $anonymous_function .= "}";
- }
- $anonymous_function .= "return 0;";
- $compare_function = create_function("\$first, \$second", $anonymous_function);
- usort($array, $compare_function);
- return $array;
+ $arguments = func_get_args();
+ $unsorted_array = array_shift($arguments);
+ $sort_order = $arguments;
+ uasort($unsorted_array, function($first_value, $second_value) use($sort_order) {
+ $result = 0;
+ for($argument_num = 1, $num_of_arguments = count($sort_order); $argument_num <= $num_of_arguments; $argument_num += 1) {
+ $key = $sort_order[$argument_num - 1];
+ $first_compare_value = $first_value[$key];
+ $second_compare_value = $second_value[$key];
+ $comparison = strcoll($first_compare_value, $second_compare_value);
+ if(0 == $comparison) {
+ $temp_result = 0;
+ } else {
+ $temp_result = (0 > $comparison) ? -1 : 1;
+ }
+ $result += $temp_result;
+ $result = 10 * $result;
+ }
+ return $result;
+ });
+ return $unsorted_array;
}
/**
@@ -149,7 +155,7 @@ class Downloads
require $cache_file;
} else {
- $data = file('http://mirrors.mageia.org/api/mageia.7.i586.list');
+ $data = file('http://mirrors.mageia.org/api/mageia.9.i586.list');
$mirrors = array();
$num_up = 0;
$num_dn = 0;
@@ -162,50 +168,66 @@ class Downloads
include_once(realpath(__DIR__) . '/../en/downloads/get/lib.php');
$refresh_country_and_city_arrays = true;
}
+ $num_of_all_mirrs = count($data);
+ $num_of_tested_mirrs = 0;
+ $common_patern = '/distrib/8/i586';
+ $common_patern = '/distrib/9/i586';
+ //~ $common_patern = '/distrib/cauldron/i586';
+ // this huge regex magic achieved with a lot of help from great https://regex101.com/
+ $single_mirror_parsing_regex = '/\s*continent\s*=\s*(?<continent>\w*)\s*,\s*';
+ $single_mirror_parsing_regex .= '\s*zone\s*=\s*(?<zone>\w*)\s*,\s*';
+ $single_mirror_parsing_regex .= '\s*country\s*=\s*(?<country>\w*)\s*,\s*';
+ $single_mirror_parsing_regex .= '(?:\s*city\s*=\s*(?<city>[\S ,]*)\s*,\s*)?';
+ $single_mirror_parsing_regex .= '\s*latitude\s*=\s*[-]?\d*\.?\d*\s*,\s*';
+ $single_mirror_parsing_regex .= '\s*longitude\s*=\s*[-]?\d*\.?\d*\s*,\s*';
+ $single_mirror_parsing_regex .= '\s*version\s*=\s*\w*\s*,\s*arch\s*=\s*\w*\s*,\s*type\s*=\s*\w*\s*,\s*';
+ $single_mirror_parsing_regex .= '\s*url\s*=\s*(?<url>\S*)\s*/m';
foreach ($data as $line) {
- $line = explode(',', trim($line));
- $m = array();
- foreach ($line as $val) {
- $val = explode('=', trim($val));
- if (!empty($val[1])) {
- $m[$val[0]] = $val[1];
- } else {
- // workaround to add second values with unescaped comma in csv line to the previous (valid) value
- end($m);
- $m[key($m)] = $m[key($m)] . ", " . $val[0];
- if ($refresh_country_and_city_arrays == true) {
- // store $line to examine faults
- $faults[] = $line;
- }
- }
+ $num_of_tested_mirrs++;
+ $mirrs_processed = sprintf("%.0f %%", $num_of_tested_mirrs / $num_of_all_mirrs * 100);
+ $regex_error = preg_match_all($single_mirror_parsing_regex, $line, $matches, PREG_SET_ORDER, 0);
+ if (false === $regex_error) {
+ $faults['error: regex parse failed'][] = $line;
+ echo 'Regex parse failed error in line ' . $line;
+ continue;
+ }
+ $m = $matches[0];
+ if ('' == $m['url']) {
+ $faults['error: parse url'][] = $line;
+ echo 'Url parse error in line ' . $line;
+ continue;
+ }
+ if ('' == $m['continent']) {
+ $faults['error: parse continent'][] = $line;
+ echo 'Continent parse error in line ' . $line;
+ continue;
+ } else {
+ $mirr_continent = $m['continent'];
+ }
+ if (false === strpos($m['url'], $common_patern)) {
+ $faults['error: url distrib missing'][] = $line;
+ echo "Url no $common_patern error in line $line";
+ continue;
}
$pu = parse_url($m['url']);
if (in_array($pu['scheme'], array('http', 'https', 'ftp'))) {
$item = array(
- 'zone' => isset($m['zone']) ? $m['zone'] : '?',
- 'country' => isset($m['country']) ? $m['country'] : '?',
- 'city' => isset($m['city']) ? $m['city'] : '-',
+ 'zone' => ('' != $m['zone']) ? $m['zone'] : '?',
+ 'country' => ('' != $m['country']) ? $m['country'] : '?',
+ 'city' => ('' != $m['city']) ? trim($m['city']) : '-',
// BEWARE of the path substitution here. Must match.
- 'url' => str_replace('/distrib/7/i586', '', $m['url'])
+ 'url' => str_replace($common_patern, '', $m['url'])
);
if ($refresh_country_and_city_arrays == true) {
// prepare details for i18n
- if (isset($m['city'])) {
- if (isset($cities_i18n[$m['city']])) {
- $mirror_cities[$m['city']] = $cities_i18n[$m['city']];
+ if ('-' != $item['city']) {
+ if (isset($cities_i18n[$item['city']])) {
+ $mirror_cities[$item['city']] = $cities_i18n[$item['city']];
} else {
- $mirror_cities[$m['city']] = '_r("' . $m['city'] . '") // new city (not yet in /en/downloads/get/lib.php)';
+ $mirror_cities[$item['city']] = '_r("' . $item['city'] . '") // new city (not yet in /en/downloads/get/lib.php)';
}
- } else if ($refresh_country_and_city_arrays == true) {
- // store data in $faults array to easy find location manually with latitude and longitude if needed
- $latitude = $m['latitude'];
- $longitude = $m['longitude'];
- $map_magnification = 5;
- $map_data = "set latitude is $latitude and longitude $longitude";
- $map_data .= " (<a href=\"https://www.openstreetmap.org/search?query=" . $latitude;
- $map_data .= "%20" . $longitude . "#map=" . $map_magnification . "/" . $latitude;
- $map_data .= "/" . $longitude . "\" >OpenStreetMap</a>)";
- $faults[$map_data] = $line;
+ } else {
+ $faults['notice: no city given'][] = $line;
}
if (isset($countries[$item['country']])) {
$mirror_countries[$item['country']] = $countries[$item['country']];
@@ -215,26 +237,27 @@ class Downloads
}
if ($documentation) {
- $test_file = $item['url'].'/doc/mga7/date.txt';
+ $test_file = $item['url'].'/doc/mga8/date.txt';
} else if ($mirrorlist) {
- $test_file = $item['url'].'/distrib/7/x86_64/media/core/updates/repodata/repomd.xml';
+ $test_file = $item['url'].'/distrib/9/x86_64/media/core/updates/repodata/repomd.xml'; // when changing, please change $common_patern too
+ $test_file = $item['url'].'/distrib/9/x86_64/media/core/release/meta-task-9-2.mga9.noarch.rpm'; // when changing, please change $common_patern too
} else {
- $test_file = $item['url'].'/iso/7.1/torrents/Mageia-7.1-x86_64.torrent';
+ $test_file = $item['url'].'/iso/9/torrents/Mageia-9-Live-Xfce-i586.torrent';
}
if (false === @file_get_contents($test_file)) {
$num_dn++;
- echo "Down ($num_dn) $test_file \n";
+ echo "Down $num_dn (up: $num_up, about $mirrs_processed of _all_ mirrors tested) $test_file \n";
} else {
$num_up++;
- echo "Up ($num_up) $test_file \n";
-// $mirrors[$m['country']][] = $item;
- $mirrors['_C:' . $m['continent']][] = $item;
+ echo "Up $num_up (down: $num_dn, about $mirrs_processed of _all_ mirrors tested) $test_file \n";
+ $mirrors['_C:' . $mirr_continent][] = $item;
}
}
}
ksort($mirrors);
foreach ($mirrors as &$continent) {
- $continent = self::sort_2d_array_by_multiple_keys($continent, 'zone', 'country', 'city', 'url');
+ $sorted_continent = self::sort_2d_array_by_multiple_keys($continent, 'zone', 'country', 'city', 'url');
+ $continent = array_values($sorted_continent);
}
unset($continent);
@@ -260,7 +283,7 @@ class Downloads
var_export($mirror_cities);
echo ";\nArray of countries for i18n: ";
var_export($countries_u);
- echo ";\nArray of faults: ";
+ echo ";\nArray of faults and warnings: ";
var_export($faults);
echo ";\n";
}
@@ -285,7 +308,7 @@ class Downloads
*
* @return array
*/
- function get_mirror($country, $continent = null, $prod = true, $documentation = false, $mirrorlist = false)
+ function get_mirror($country, $continent = null, $prod = true, $documentation = false, $mirrorlist = false, $https_only = false)
{
$mirs = self::get_all_mirrors($prod, $documentation, $mirrorlist);
$continent = '_C:' . $continent;
@@ -297,67 +320,136 @@ class Downloads
'continent_minus_country' => array(),
'other_continents' => array()
);
+ $hostnames = array();
+ $list_of_mirrs['country_alternate_protocol'] = array();
+ $continent_https_only = array();
+ $other_continents_https_only = array();
foreach ($mirs as $curr_continent => $continent_mirrors) {
- if ($continent != $curr_continent)
- {
- if ($mirrorlist)
+ foreach ($continent_mirrors as $mirror) {
+ if ($continent != $curr_continent)
{
- foreach ($continent_mirrors as $mirror) {
+ if ($mirrorlist)
+ {
$list_of_mirrs['other_continents'][] = $mirror['url'];
}
+ elseif (true === $https_only)
+ {
+ $pu = parse_url($mirror['url']);
+ if ('https' == $pu['scheme'])
+ {
+ $other_continents_https_only[$curr_continent][] = $mirror;
+ }
+ }
}
- continue;
- }
- foreach ($continent_mirrors as $mirror) {
- if (strpos($mirror['url'], 'distrib-coffee.ipsl.jussieu.fr') !== false)
- {
- // exclude source server to drop it's DL load
- continue;
- }
- // keep assisting the french mirrors with german ones
- if ($mirror['country'] == 'DE')
- {
- $fr_mirr_asist[] = $mirror;
- }
- // only add german mirrors when french are on turn
- // sorting of mirror db cache must be kept to work properly (DE before FR)
- if ($country == 'FR' && $mirror['country'] == 'FR' && count($fr_mirr_asist) > 0)
- {
- $mirrors[$continent] = $fr_mirr_asist;
- $fr_mirr_asist = array();
- }
- if ($mirrorlist)
+ else
{
- if ($mirror['country'] == $country)
+ if (strpos($mirror['url'], 'distrib-coffee.ipsl.jussieu.fr') !== false)
{
- $list_of_mirrs['country'][] = $mirror['url'];
+ // exclude source server to drop it's DL load
+ continue;
+ }
+ // keep assisting the french mirrors with german ones
+ if ($country == 'FR' && $mirror['country'] == 'DE')
+ {
+ $pu = parse_url($mirror['url']);
+ if ('https' == $pu['scheme']) {
+ $fr_mirr_asist[] = $mirror;
+ }
+ }
+ // only add german mirrors when french are on turn
+ // sorting of mirror db cache must be kept to work properly (DE before FR)
+ if ($country == 'FR' && $mirror['country'] == 'FR' && count($fr_mirr_asist) > 0)
+ {
+ $mirrors[$continent] = $fr_mirr_asist;
+ $fr_mirr_asist = array();
+ }
+ if ($mirrorlist)
+ {
+ $pu = parse_url($mirror['url']);
+ if (in_array($pu['host'], $hostnames))
+ {
+ $list_of_mirrs['country_alternate_protocol'][] = $mirror['url'];
+ }
+ else
+ {
+ $hostnames[] = $pu['host'];
+ if ($mirror['country'] == $country)
+ {
+ $list_of_mirrs['country'][] = $mirror['url'];
+ }
+ else
+ {
+ $list_of_mirrs['continent_minus_country'][] = $mirror['url'];
+ }
+ }
}
else
{
- $list_of_mirrs['continent_minus_country'][] = $mirror['url'];
+ if ($mirror['country'] == $country)
+ {
+ if (true === $https_only)
+ {
+ $pu = parse_url($mirror['url']);
+ if ('https' == $pu['scheme']) {
+ $mirrors[$continent][] = $mirror;
+ }
+ }
+ else
+ {
+ $mirrors[$continent][] = $mirror;
+ }
+ }
+ else
+ {
+ if (true === $https_only)
+ {
+ $pu = parse_url($mirror['url']);
+ if ('https' == $pu['scheme']) {
+ $continent_https_only[$curr_continent][] = $mirror;
+ }
+ }
+ }
}
}
- else
+ }
+ }
+ if (count($mirrors) == 0)
+ {
+ if (true === $https_only)
+ {
+ if (count($continent_https_only) > 0)
{
- if ($mirror['country'] == $country)
- {
- $mirrors[$continent][] = $mirror;
- }
+ $mirrors = $continent_https_only;
+ }
+ if (count($mirrors) == 0 && count($other_continents_https_only) > 0)
+ {
+ $mirrors = $other_continents_https_only;
}
}
- if (count($mirrors) == 0)
+ else
{
// add all continent mirrors if country doesn't have any
$mirrors[$continent] = $continent_mirrors;
}
}
+ // falback if selection fails
if (count($mirrors) > 0)
{
$mirs = $mirrors;
}
- shuffle($mirs);
- $mirr_continent = $mirs[0];
+ // a workaround as shuffle() doesn't preserve assoc. keys
+ if (count($mirs) > 1)
+ {
+ $mirs_keys = array_keys($mirs);
+ shuffle($mirs_keys);
+ foreach($mirs_keys as $key) {
+ $shuffled[$key] = $mirs[$key];
+ }
+ $mirs = $shuffled;
+ }
+
+ $mirr_continent = array_keys($mirs)[0];
$mirs = array_shift($mirs);
shuffle($mirs);
$one_mirror = array_shift($mirs);
@@ -371,6 +463,7 @@ class Downloads
$resulting_mirrs = array_merge(
$list_of_mirrs['country'],
$list_of_mirrs['continent_minus_country'],
+ $list_of_mirrs['country_alternate_protocol'],
$list_of_mirrs['other_continents']
);
$one_mirror['mirrors_list'] = array_slice($resulting_mirrs, 0, 10);
@@ -379,9 +472,9 @@ class Downloads
return $one_mirror;
}
- function prepare_download($force = false, $country = null, $prod = true, $documentation = false, $mirrorlist = false)
+ function prepare_download($force = false, $country = null, $prod = true, $documentation = false, $mirrorlist = false, $https_only = false)
{
- return $this->get_one_mirror($force, $country, $prod, $documentation, $mirrorlist);
+ return $this->get_one_mirror($force, $country, $prod, $documentation, $mirrorlist, $https_only);
}
/**
@@ -397,7 +490,7 @@ class Downloads
*
* TODO extract as much as possible $_SESSION(read) and $_SERVER and $_GET
*/
- function get_one_mirror($force = false, $country = null, $prod = true, $documentation = false, $mirrorlist = false)
+ function get_one_mirror($force = false, $country = null, $prod = true, $documentation = false, $mirrorlist = false, $https_only = false)
{
$fuzzy_mirror = false;
@@ -459,7 +552,7 @@ class Downloads
$continent = null;
}
- $mirror = $this->get_mirror($country, $continent, $prod, $documentation, $mirrorlist);
+ $mirror = $this->get_mirror($country, $continent, $prod, $documentation, $mirrorlist, $https_only);
$mirror['purl'] = parse_url($mirror['url']);
// reassign country, as get_one_mirror() may have decided