optimise mirror db cache size (half in size)

+ with added benefit of cleaner git diff
+ verbosity improvements of mirror db cache generate tool

1 files changed, 86 insertions, 23 deletions

diff --git a/lib/Downloads.php b/lib/Downloads.php
index 8b7a74fce..b69bafaea 100644
--- a/lib/Downloads.php
+++ b/lib/Downloads.php
@@ -86,6 +86,41 @@ class Downloads
     }
 
     /**
+     * Sort 2D array by multiple associative or numeric keys.
+     * $sorted_array = self::sort_2d_array_by_multiple_keys($unsorted_array, 'first key', 'second', ...);
+     *
+     * based on SortArray http://php.net/manual/en/function.usort.php#42535
+     *
+     * @param array $unsorted_array
+     *
+     * @param string first key to order by
+     *
+     * @param string second key to order by
+     *
+     * @param string add as many keys to order by as needed
+     *
+     * @return array $sorted_array
+    */
+    public static function sort_2d_array_by_multiple_keys()
+    {
+        $arguments = func_get_args();
+        $array = $arguments[0];
+        $anonymous_function = '';
+        $num_of_arguments = count($arguments);
+        for ($cur_argument = 1; $cur_argument < $num_of_arguments; $cur_argument++) {
+            $anonymous_function .= "if (\$first['$arguments[$cur_argument]'] != \$second['$arguments[$cur_argument]']) {";
+            $anonymous_function .= "    \$compare_result = strcoll(\$first['$arguments[$cur_argument]'], \$second['$arguments[$cur_argument]']);";
+            $anonymous_function .= "    if (0 == \$compare_result) { return 0; };";
+            $anonymous_function .= "    return ((0 > \$compare_result) ? -1 : 1);";
+            $anonymous_function .= "}";
+        }
+        $anonymous_function .= "return 0;";
+        $compare_function = create_function("\$first, \$second", $anonymous_function);
+        usort($array, $compare_function);
+        return $array;
+    }
+
+    /**
      * Get mirrors list from mirrors.mageia.org,
      * store/cache it in a different key/value format
      * (keys are: "$country" and "_C:$continent"),
@@ -110,6 +145,8 @@ class Downloads
         } else {
             $data    = file('http://mirrors.mageia.org/api/mageia.5.i586.list');
             $mirrors = array();
+            $num_up  = 0;
+            $num_dn  = 0;
             foreach ($data as $line) {
                 $line = explode(',', trim($line));
                 $m    = array();
@@ -122,9 +159,9 @@ class Downloads
                 $pu = parse_url($m['url']);
                 if (in_array($pu['scheme'], array('http', 'https', 'ftp'))) {
                     $item = array(
+                        'zone'      => isset($m['zone']) ? $m['zone'] : '?',
+                        'country'   => isset($m['country']) ? $m['country'] : '?',
                         'city'      => isset($m['city']) ? $m['city'] : '?',
-                        'continent' => isset($m['continent']) ? $m['continent'] : '?',
-                        'zone'      => $m['zone'],
                         // BEWARE of the path substitution here. Must match.
                         'url'       => str_replace('/distrib/5/i586', '', $m['url'])
                     );
@@ -134,16 +171,24 @@ class Downloads
                     } else {
                         $test_file = $item['url'].'/iso/5/torrents/Mageia-5-LiveDVD-KDE4-x86_64-DVD.torrent';
                     }
-                    if (false === file_get_contents($test_file)) {
-                        echo "down $test_file \n";
+                    if (false === @file_get_contents($test_file)) {
+                        $num_dn++;
+                        echo "Down ($num_dn) $test_file \n";
                     } else {
-                        echo "Up $test_file \n";
-                        $mirrors[$m['country']][]           = $item;
+                        $num_up++;
+                        echo "Up ($num_up) $test_file \n";
+//                         $mirrors[$m['country']][]           = $item;
                         $mirrors['_C:' . $m['continent']][] = $item;
                     }
                 }
             }
+            ksort($mirrors);
+            foreach ($mirrors as &$continent) {
+                $continent = self::sort_2d_array_by_multiple_keys($continent, 'zone', 'country', 'city', 'url');
+            }
+            unset($continent);
 
+            echo "\nThere are $num_up servers with the file and $num_dn with some kind of issue.\n";
             file_put_contents($cache_file,
                 sprintf('<?php $mirrors = %s; ?>' . PHP_EOL, var_export($mirrors, true)));
         }
@@ -152,10 +197,10 @@ class Downloads
     }
 
     /**
-     * Get mirrors from stored dictionary and find:
-     * - best matching country
-     * - or best matching continent
-     * - or random
+     * Get mirrors from stored dictionary and find best matching mirror:
+     * - if it exists in the country otherwise
+     * - on continent if it exists otherwise
+     * - random mirror
      *
      * @param string $country
      * @param string $continent
@@ -167,27 +212,45 @@ class Downloads
         $mirs      = self::get_all_mirrors();
         $continent = '_C:' . $continent;
 
-        if (array_key_exists($country, $mirs))
-        {
-            $mirs_tmp = $mirs[$country];
-            if ($country == 'FR')
+        $mirrors = array();
+        $fr_mirr_asist = array();
+        foreach ($mirs as $curr_continent => $continent_mirrors) {
+            if (!is_null($continent) && $continent != $curr_continent)
             {
-                $mirs_tmp = array_merge($mirs_tmp, $mirs['DE']);
+                continue;
+            }
+            foreach ($continent_mirrors as $mirror) {
+                // keep assisting the french mirrors with german ones
+                if ($mirror['country'] == 'DE')
+                {
+                    $fr_mirr_asist[] = $mirror;
+                }
+                // only add german mirrors when french are on turn
+                // sorting of mirror db cache must be kept to work properly
+                if ($country == 'FR' && $mirror['country'] == 'FR' && count($fr_mirr_asist) > 0)
+                {
+                    $mirrors[$continent] = $fr_mirr_asist;
+                    $fr_mirr_asist = array();
+                }
+                if ($mirror['country'] == $country)
+                {
+                    $mirrors[$continent][] = $mirror;
+                }
             }
-            $mirs = $mirs_tmp;
-        }
-        elseif (array_key_exists($continent, $mirs)) {
-            $mirs = $mirs[$continent];
         }
-        else
+        if (count($mirrors) > 0)
         {
-            shuffle($mirs);
-            $mirs = array_shift($mirs);
+            $mirs = $mirrors;
         }
 
         shuffle($mirs);
+        $mirr_continent = $mirs[0];
+        $mirs = array_shift($mirs);
+        shuffle($mirs);
+        $one_mirror = array_shift($mirs);
+        $one_mirror['continent'] = $mirr_continent;
 
-        return array_shift($mirs);
+        return $one_mirror;
     }
 
     function prepare_download($force = false, $country = null)