diff options
Diffstat (limited to 'lib/Downloads.php')
-rw-r--r-- | lib/Downloads.php | 295 |
1 files changed, 194 insertions, 101 deletions
diff --git a/lib/Downloads.php b/lib/Downloads.php index f4b21f85f..69a0e38ad 100644 --- a/lib/Downloads.php +++ b/lib/Downloads.php @@ -88,8 +88,8 @@ class Downloads /** * Sort 2D array by multiple associative or numeric keys. * $sorted_array = self::sort_2d_array_by_multiple_keys($unsorted_array, 'first key', 'second', ...); - * - * based on SortArray http://php.net/manual/en/function.usort.php#42535 + * + * Note that keys are preserved! * * @param array $unsorted_array * @@ -103,21 +103,27 @@ class Downloads */ public static function sort_2d_array_by_multiple_keys() { - $arguments = func_get_args(); - $array = $arguments[0]; - $anonymous_function = ''; - $num_of_arguments = count($arguments); - for ($cur_argument = 1; $cur_argument < $num_of_arguments; $cur_argument++) { - $anonymous_function .= "if (\$first['$arguments[$cur_argument]'] != \$second['$arguments[$cur_argument]']) {"; - $anonymous_function .= " \$compare_result = strcoll(\$first['$arguments[$cur_argument]'], \$second['$arguments[$cur_argument]']);"; - $anonymous_function .= " if (0 == \$compare_result) { return 0; };"; - $anonymous_function .= " return ((0 > \$compare_result) ? -1 : 1);"; - $anonymous_function .= "}"; - } - $anonymous_function .= "return 0;"; - $compare_function = create_function("\$first, \$second", $anonymous_function); - usort($array, $compare_function); - return $array; + $arguments = func_get_args(); + $unsorted_array = array_shift($arguments); + $sort_order = $arguments; + uasort($unsorted_array, function($first_value, $second_value) use($sort_order) { + $result = 0; + for($argument_num = 1, $num_of_arguments = count($sort_order); $argument_num <= $num_of_arguments; $argument_num += 1) { + $key = $sort_order[$argument_num - 1]; + $first_compare_value = $first_value[$key]; + $second_compare_value = $second_value[$key]; + $comparison = strcoll($first_compare_value, $second_compare_value); + if(0 == $comparison) { + $temp_result = 0; + } else { + $temp_result = (0 > $comparison) ? -1 : 1; + } + $result += $temp_result; + $result = 10 * $result; + } + return $result; + }); + return $unsorted_array; } /** @@ -149,7 +155,7 @@ class Downloads require $cache_file; } else { - $data = file('http://mirrors.mageia.org/api/mageia.7.i586.list'); + $data = file('http://mirrors.mageia.org/api/mageia.9.i586.list'); $mirrors = array(); $num_up = 0; $num_dn = 0; @@ -162,50 +168,66 @@ class Downloads include_once(realpath(__DIR__) . '/../en/downloads/get/lib.php'); $refresh_country_and_city_arrays = true; } + $num_of_all_mirrs = count($data); + $num_of_tested_mirrs = 0; + $common_patern = '/distrib/8/i586'; + $common_patern = '/distrib/9/i586'; + //~ $common_patern = '/distrib/cauldron/i586'; + // this huge regex magic achieved with a lot of help from great https://regex101.com/ + $single_mirror_parsing_regex = '/\s*continent\s*=\s*(?<continent>\w*)\s*,\s*'; + $single_mirror_parsing_regex .= '\s*zone\s*=\s*(?<zone>\w*)\s*,\s*'; + $single_mirror_parsing_regex .= '\s*country\s*=\s*(?<country>\w*)\s*,\s*'; + $single_mirror_parsing_regex .= '(?:\s*city\s*=\s*(?<city>[\S ,]*)\s*,\s*)?'; + $single_mirror_parsing_regex .= '\s*latitude\s*=\s*[-]?\d*\.?\d*\s*,\s*'; + $single_mirror_parsing_regex .= '\s*longitude\s*=\s*[-]?\d*\.?\d*\s*,\s*'; + $single_mirror_parsing_regex .= '\s*version\s*=\s*\w*\s*,\s*arch\s*=\s*\w*\s*,\s*type\s*=\s*\w*\s*,\s*'; + $single_mirror_parsing_regex .= '\s*url\s*=\s*(?<url>\S*)\s*/m'; foreach ($data as $line) { - $line = explode(',', trim($line)); - $m = array(); - foreach ($line as $val) { - $val = explode('=', trim($val)); - if (!empty($val[1])) { - $m[$val[0]] = $val[1]; - } else { - // workaround to add second values with unescaped comma in csv line to the previous (valid) value - end($m); - $m[key($m)] = $m[key($m)] . ", " . $val[0]; - if ($refresh_country_and_city_arrays == true) { - // store $line to examine faults - $faults[] = $line; - } - } + $num_of_tested_mirrs++; + $mirrs_processed = sprintf("%.0f %%", $num_of_tested_mirrs / $num_of_all_mirrs * 100); + $regex_error = preg_match_all($single_mirror_parsing_regex, $line, $matches, PREG_SET_ORDER, 0); + if (false === $regex_error) { + $faults['error: regex parse failed'][] = $line; + echo 'Regex parse failed error in line ' . $line; + continue; + } + $m = $matches[0]; + if ('' == $m['url']) { + $faults['error: parse url'][] = $line; + echo 'Url parse error in line ' . $line; + continue; + } + if ('' == $m['continent']) { + $faults['error: parse continent'][] = $line; + echo 'Continent parse error in line ' . $line; + continue; + } else { + $mirr_continent = $m['continent']; + } + if (false === strpos($m['url'], $common_patern)) { + $faults['error: url distrib missing'][] = $line; + echo "Url no $common_patern error in line $line"; + continue; } $pu = parse_url($m['url']); if (in_array($pu['scheme'], array('http', 'https', 'ftp'))) { $item = array( - 'zone' => isset($m['zone']) ? $m['zone'] : '?', - 'country' => isset($m['country']) ? $m['country'] : '?', - 'city' => isset($m['city']) ? $m['city'] : '-', + 'zone' => ('' != $m['zone']) ? $m['zone'] : '?', + 'country' => ('' != $m['country']) ? $m['country'] : '?', + 'city' => ('' != $m['city']) ? trim($m['city']) : '-', // BEWARE of the path substitution here. Must match. - 'url' => str_replace('/distrib/7/i586', '', $m['url']) + 'url' => str_replace($common_patern, '', $m['url']) ); if ($refresh_country_and_city_arrays == true) { // prepare details for i18n - if (isset($m['city'])) { - if (isset($cities_i18n[$m['city']])) { - $mirror_cities[$m['city']] = $cities_i18n[$m['city']]; + if ('-' != $item['city']) { + if (isset($cities_i18n[$item['city']])) { + $mirror_cities[$item['city']] = $cities_i18n[$item['city']]; } else { - $mirror_cities[$m['city']] = '_r("' . $m['city'] . '") // new city (not yet in /en/downloads/get/lib.php)'; + $mirror_cities[$item['city']] = '_r("' . $item['city'] . '") // new city (not yet in /en/downloads/get/lib.php)'; } - } else if ($refresh_country_and_city_arrays == true) { - // store data in $faults array to easy find location manually with latitude and longitude if needed - $latitude = $m['latitude']; - $longitude = $m['longitude']; - $map_magnification = 5; - $map_data = "set latitude is $latitude and longitude $longitude"; - $map_data .= " (<a href=\"https://www.openstreetmap.org/search?query=" . $latitude; - $map_data .= "%20" . $longitude . "#map=" . $map_magnification . "/" . $latitude; - $map_data .= "/" . $longitude . "\" >OpenStreetMap</a>)"; - $faults[$map_data] = $line; + } else { + $faults['notice: no city given'][] = $line; } if (isset($countries[$item['country']])) { $mirror_countries[$item['country']] = $countries[$item['country']]; @@ -215,26 +237,27 @@ class Downloads } if ($documentation) { - $test_file = $item['url'].'/doc/mga7/date.txt'; + $test_file = $item['url'].'/doc/mga8/date.txt'; } else if ($mirrorlist) { - $test_file = $item['url'].'/distrib/7/x86_64/media/core/updates/repodata/repomd.xml'; + $test_file = $item['url'].'/distrib/9/x86_64/media/core/updates/repodata/repomd.xml'; // when changing, please change $common_patern too + $test_file = $item['url'].'/distrib/9/x86_64/media/core/release/meta-task-9-2.mga9.noarch.rpm'; // when changing, please change $common_patern too } else { - $test_file = $item['url'].'/iso/7.1/torrents/Mageia-7.1-x86_64.torrent'; + $test_file = $item['url'].'/iso/9/torrents/Mageia-9-Live-Xfce-i586.torrent'; } if (false === @file_get_contents($test_file)) { $num_dn++; - echo "Down ($num_dn) $test_file \n"; + echo "Down $num_dn (up: $num_up, about $mirrs_processed of _all_ mirrors tested) $test_file \n"; } else { $num_up++; - echo "Up ($num_up) $test_file \n"; -// $mirrors[$m['country']][] = $item; - $mirrors['_C:' . $m['continent']][] = $item; + echo "Up $num_up (down: $num_dn, about $mirrs_processed of _all_ mirrors tested) $test_file \n"; + $mirrors['_C:' . $mirr_continent][] = $item; } } } ksort($mirrors); foreach ($mirrors as &$continent) { - $continent = self::sort_2d_array_by_multiple_keys($continent, 'zone', 'country', 'city', 'url'); + $sorted_continent = self::sort_2d_array_by_multiple_keys($continent, 'zone', 'country', 'city', 'url'); + $continent = array_values($sorted_continent); } unset($continent); @@ -260,7 +283,7 @@ class Downloads var_export($mirror_cities); echo ";\nArray of countries for i18n: "; var_export($countries_u); - echo ";\nArray of faults: "; + echo ";\nArray of faults and warnings: "; var_export($faults); echo ";\n"; } @@ -285,7 +308,7 @@ class Downloads * * @return array */ - function get_mirror($country, $continent = null, $prod = true, $documentation = false, $mirrorlist = false) + function get_mirror($country, $continent = null, $prod = true, $documentation = false, $mirrorlist = false, $https_only = false) { $mirs = self::get_all_mirrors($prod, $documentation, $mirrorlist); $continent = '_C:' . $continent; @@ -297,67 +320,136 @@ class Downloads 'continent_minus_country' => array(), 'other_continents' => array() ); + $hostnames = array(); + $list_of_mirrs['country_alternate_protocol'] = array(); + $continent_https_only = array(); + $other_continents_https_only = array(); foreach ($mirs as $curr_continent => $continent_mirrors) { - if ($continent != $curr_continent) - { - if ($mirrorlist) + foreach ($continent_mirrors as $mirror) { + if ($continent != $curr_continent) { - foreach ($continent_mirrors as $mirror) { + if ($mirrorlist) + { $list_of_mirrs['other_continents'][] = $mirror['url']; } + elseif (true === $https_only) + { + $pu = parse_url($mirror['url']); + if ('https' == $pu['scheme']) + { + $other_continents_https_only[$curr_continent][] = $mirror; + } + } } - continue; - } - foreach ($continent_mirrors as $mirror) { - if (strpos($mirror['url'], 'distrib-coffee.ipsl.jussieu.fr') !== false) - { - // exclude source server to drop it's DL load - continue; - } - // keep assisting the french mirrors with german ones - if ($mirror['country'] == 'DE') - { - $fr_mirr_asist[] = $mirror; - } - // only add german mirrors when french are on turn - // sorting of mirror db cache must be kept to work properly (DE before FR) - if ($country == 'FR' && $mirror['country'] == 'FR' && count($fr_mirr_asist) > 0) - { - $mirrors[$continent] = $fr_mirr_asist; - $fr_mirr_asist = array(); - } - if ($mirrorlist) + else { - if ($mirror['country'] == $country) + if (strpos($mirror['url'], 'distrib-coffee.ipsl.jussieu.fr') !== false) { - $list_of_mirrs['country'][] = $mirror['url']; + // exclude source server to drop it's DL load + continue; + } + // keep assisting the french mirrors with german ones + if ($country == 'FR' && $mirror['country'] == 'DE') + { + $pu = parse_url($mirror['url']); + if ('https' == $pu['scheme']) { + $fr_mirr_asist[] = $mirror; + } + } + // only add german mirrors when french are on turn + // sorting of mirror db cache must be kept to work properly (DE before FR) + if ($country == 'FR' && $mirror['country'] == 'FR' && count($fr_mirr_asist) > 0) + { + $mirrors[$continent] = $fr_mirr_asist; + $fr_mirr_asist = array(); + } + if ($mirrorlist) + { + $pu = parse_url($mirror['url']); + if (in_array($pu['host'], $hostnames)) + { + $list_of_mirrs['country_alternate_protocol'][] = $mirror['url']; + } + else + { + $hostnames[] = $pu['host']; + if ($mirror['country'] == $country) + { + $list_of_mirrs['country'][] = $mirror['url']; + } + else + { + $list_of_mirrs['continent_minus_country'][] = $mirror['url']; + } + } } else { - $list_of_mirrs['continent_minus_country'][] = $mirror['url']; + if ($mirror['country'] == $country) + { + if (true === $https_only) + { + $pu = parse_url($mirror['url']); + if ('https' == $pu['scheme']) { + $mirrors[$continent][] = $mirror; + } + } + else + { + $mirrors[$continent][] = $mirror; + } + } + else + { + if (true === $https_only) + { + $pu = parse_url($mirror['url']); + if ('https' == $pu['scheme']) { + $continent_https_only[$curr_continent][] = $mirror; + } + } + } } } - else + } + } + if (count($mirrors) == 0) + { + if (true === $https_only) + { + if (count($continent_https_only) > 0) { - if ($mirror['country'] == $country) - { - $mirrors[$continent][] = $mirror; - } + $mirrors = $continent_https_only; + } + if (count($mirrors) == 0 && count($other_continents_https_only) > 0) + { + $mirrors = $other_continents_https_only; } } - if (count($mirrors) == 0) + else { // add all continent mirrors if country doesn't have any $mirrors[$continent] = $continent_mirrors; } } + // falback if selection fails if (count($mirrors) > 0) { $mirs = $mirrors; } - shuffle($mirs); - $mirr_continent = $mirs[0]; + // a workaround as shuffle() doesn't preserve assoc. keys + if (count($mirs) > 1) + { + $mirs_keys = array_keys($mirs); + shuffle($mirs_keys); + foreach($mirs_keys as $key) { + $shuffled[$key] = $mirs[$key]; + } + $mirs = $shuffled; + } + + $mirr_continent = array_keys($mirs)[0]; $mirs = array_shift($mirs); shuffle($mirs); $one_mirror = array_shift($mirs); @@ -371,6 +463,7 @@ class Downloads $resulting_mirrs = array_merge( $list_of_mirrs['country'], $list_of_mirrs['continent_minus_country'], + $list_of_mirrs['country_alternate_protocol'], $list_of_mirrs['other_continents'] ); $one_mirror['mirrors_list'] = array_slice($resulting_mirrs, 0, 10); @@ -379,9 +472,9 @@ class Downloads return $one_mirror; } - function prepare_download($force = false, $country = null, $prod = true, $documentation = false, $mirrorlist = false) + function prepare_download($force = false, $country = null, $prod = true, $documentation = false, $mirrorlist = false, $https_only = false) { - return $this->get_one_mirror($force, $country, $prod, $documentation, $mirrorlist); + return $this->get_one_mirror($force, $country, $prod, $documentation, $mirrorlist, $https_only); } /** @@ -397,7 +490,7 @@ class Downloads * * TODO extract as much as possible $_SESSION(read) and $_SERVER and $_GET */ - function get_one_mirror($force = false, $country = null, $prod = true, $documentation = false, $mirrorlist = false) + function get_one_mirror($force = false, $country = null, $prod = true, $documentation = false, $mirrorlist = false, $https_only = false) { $fuzzy_mirror = false; @@ -459,7 +552,7 @@ class Downloads $continent = null; } - $mirror = $this->get_mirror($country, $continent, $prod, $documentation, $mirrorlist); + $mirror = $this->get_mirror($country, $continent, $prod, $documentation, $mirrorlist, $https_only); $mirror['purl'] = parse_url($mirror['url']); // reassign country, as get_one_mirror() may have decided |