report about differences between Transifex and our git repository for documentation added

2 files changed, 155 insertions, 61 deletions

diff --git a/langs/lib.php b/langs/lib.php
index 03520e317..3b68db368 100644
--- a/langs/lib.php
+++ b/langs/lib.php
@@ -81,24 +81,40 @@ function _lang_diff($a, $b)
  *
  * @return array
 */
-function _po_diff($locale, $resource)
+function _po_diff($locale, $resource, $path = NULL)
 {
-    $source_l = read_translation_file('en', $resource);
-    $target_l = read_translation_file($locale, $resource);
+    if (is_null($path)) {
+        $source_l = read_translation_file('en', $resource);
+        $target_l = read_translation_file($locale, $resource);
+    } else {
+        $source_path_filename = sprintf('%s/%s.pot', $path, $resource);
+        $source_l = phpmo_parse_po_file($source_path_filename);
+        if ('en' == $locale) {
+            $target_l = $source_l;
+        } else {
+            $locale = locale_hyphen_underscore($locale, true);
+            $target_path_filename = sprintf('%s/%s.po', $path, $locale);
+            $target_l = phpmo_parse_po_file($target_path_filename);
+        }
+    }
 
     $pot_strings = array();
     $untrans = array();
     $fuzzy_or_missing = array();
 
-    foreach ($source_l as $escaped_string => $subarray) {
-        if (!empty($subarray["msgid"])) { // filter out header
-            $pot_strings[$escaped_string] = $subarray["msgid"];
+    if ($source_l != FALSE) {
+        foreach ($source_l as $escaped_string => $subarray) {
+            if (!empty($subarray["msgid"])) { // filter out header
+                $pot_strings[$escaped_string] = $subarray["msgid"];
+            }
         }
     }
 
-    foreach ($target_l as $escaped_string => $subarray) {
-        if (!empty($subarray["msgid"])) { // filter out header
-            $po_strings[$escaped_string] = $subarray["msgstr"][0];
+    if ($target_l != FALSE) {
+        foreach ($target_l as $escaped_string => $subarray) {
+            if (!empty($subarray["msgid"])) { // filter out header
+                $po_strings[$escaped_string] = $subarray["msgstr"][0];
+            }
         }
     }
 
@@ -113,12 +129,13 @@ function _po_diff($locale, $resource)
     }
 
     return array(
-        'a'       => count($pot_strings), // # of original strings
-//        'b'       => count($po_strings), // # of target strings
-        'fuzzy_or_missing' => $fuzzy_or_missing,
-        'notrans' => $untrans,
-        'extra'   => array(),
-        'dup_str' => array(),
+        'a'                => count($pot_strings), // # of original strings
+//      'b'                => count($po_strings), // # of target strings
+//      'source_strings'   => $pot_strings, // array of original strings
+        'fuzzy_or_missing' => $fuzzy_or_missing, // array of fuzzy or missing strings
+        'notrans'          => $untrans, // array of untranslated strings
+        'extra'            => array(),
+        'dup_str'          => array(),
     );
 }
 
diff --git a/langs/report_tx_git.php b/langs/report_tx_git.php
index edc18ccb9..846f282ed 100644
--- a/langs/report_tx_git.php
+++ b/langs/report_tx_git.php
@@ -29,14 +29,35 @@
     </header>
     <div id="page">
 
-    <p>Progress report is available <a href="./report.php">here</a>.</p>
+    <p>Languages translation progress report for web pages is available <a href="./report.php">here</a>.</p>
 
 <?php
 define('HLANG', TRUE);
 include 'lib.php';
 
 $errors = array(); // stored for error management ;)
-$resource_type = 'Webpages'; // Webpages, Documentation, Cauldron
+$c = (isset($_GET['c']) ? strip_tags(trim($_GET['c'])) : '');
+if (in_array($c, array('Webpages', 'Documentation', 'Cauldron'))) {
+    $resource_type = $c; // filter only valid resource types
+} else {
+    $resource_type = 'Webpages'; // default
+}
+
+if ('Documentation' == $resource_type) {
+    $git_resources = array(
+        array(
+            'pot_name' => 'help', // Mageia Installer Help
+            'tx_name'  => 'doc_installer',
+            'git_path' => 'http://gitweb.mageia.org/software/i18n/tools/plain/docs/installer',
+        ),
+        array(
+            'pot_name' => 'mcc-help', // Mageia Control Center Help
+            'tx_name'  => 'mcc-help',
+            'git_path' => 'http://gitweb.mageia.org/software/i18n/tools/plain/docs/mcc-help',
+        ),
+    );
+}
+
 
 /**
  * Transifex API implementation in php detailed on http://docs.transifex.com/developer/api/projects
@@ -90,21 +111,73 @@ function build_links($git_resource_name, $language_code, $resource_type, $stat_d
     $tx_resource_name = str_replace('/', '-', $git_resource_name);
     $locale_name = locale_underscore_to_hyphen($language_code);
     if ('nav' == $git_resource_name) {
-        $git_link = sprintf('nav/tree/langs/%s.po', $locale_name);
+        $git_link = sprintf('http://gitweb.mageia.org/web/nav/tree/langs/%s.po', $locale_name);
+    } else if ('Documentation' == $resource_type) {
+        $git_link = sprintf('%s/%s.po', $stat_data['git_path'], $language_code);
     } else {
-        $git_link = sprintf('www/tree/langs/%s/%s.po', $locale_name, $git_resource_name);
+        $git_link = sprintf('http://gitweb.mageia.org/web/www/tree/langs/%s/%s.po', $locale_name, $git_resource_name);
     }
     $links_and_num  = "<a href=\"http://www.transifex.com/organization/MageiaLinux/dashboard/all_resources/";
     $links_and_num .= "$language_code/#1/?s=$tx_resource_name&amp;c=$resource_type\">Tx</a>: ";
     $links_and_num .= $stat_data['tx_untran'];
     $links_and_num .= (0 == $stat_data['tx_untran'] ? ' - full' : '');
-    $links_and_num .= ', <a href="http://gitweb.mageia.org/web/' . $git_link . '">git</a>: ';
+    $links_and_num .= ', <a href="' . $git_link . '">git</a>: ';
     $links_and_num .= $stat_data['git_untran'];
     $links_and_num .= ($stat_data['num_of_all'] == $stat_data['git_untran'] ? ' - empty' : '');
 
     return $links_and_num;
 }
 
+/**
+ * Generating report about git resources statistics
+ *
+ * @param array  $language_codes list
+ * @param array  $resource_names list
+ * @param string $path to the git, otherwise NULL
+ *
+ * @return array
+*/
+function generating_report($language_codes, $resource_names, $path = NULL, $pot_name = NULL)
+{
+    $report = array();
+    foreach ($language_codes as $l) {
+        foreach ($resource_names as $f) {
+            $references = '';
+            if (is_null($pot_name)) {
+                $resource = _extract_resource($f);
+            } else {
+                $resource = $pot_name;
+            }
+            $langF    = _po_file_switch($f, $l);
+            if (strstr($f, '../_nav/langs/en.') !== FALSE) {
+                $langF = '../_nav/langs/' . $l . '.po' . (($l == 'en') ? 't' : '');
+            }
+            if (!is_null($path) || file_exists($langF)) {
+                $stat = _po_diff($l, $resource, $path);
+                $num_of_fuzzy_or_missing = count($stat['fuzzy_or_missing']);
+                $num_of_untranslated     = count($stat['notrans']);
+            } else { // file $langF doesn't exists in 'Webpages' $resource_type
+                $stat = _po_diff('en', $resource);
+                $num_of_fuzzy_or_missing = 0;
+                $num_of_untranslated     = count($stat['notrans']);
+            }
+            // unify resource names, navigation is a special exception
+            $resource_name = str_replace(array('../_nav/langs/en', 'en/', '.pot'), array('nav', '', ''), $f);
+            // create pt_BR from pt-br and alike to unify languages
+            $web_language_code = locale_hyphen_underscore($l, true);
+            $num_of_not_fully_trans = $num_of_fuzzy_or_missing + $num_of_untranslated;
+            $report[] = array(
+                'num_of_all_strings'     => $stat['a'],
+                'resource_name'          => $resource_name,
+                'web_language_code'      => $web_language_code,
+                'num_of_not_fully_trans' => $num_of_not_fully_trans,
+                'webgit_path'            => $path,
+            );
+        }
+    }
+    return $report;
+}
+
 
 $tx_resources = array();
 // get resources data from TX
@@ -129,36 +202,19 @@ foreach ($tx_resources as $one_resource) {
 
 // create array ($report) with statistics from git
 $report = array();
-$otherLangs = get_other_langs();
-$enFiles = array_merge(array('../_nav/langs/en.pot'), get_lang_references('*.pot')); // added navigation file
-foreach ($otherLangs as $l) {
-    foreach ($enFiles as $f) {
-        $references = '';
-        $resource = _extract_resource($f);
-        $langF    = _po_file_switch($f, $l);
-        if (strstr($f, '../_nav/langs/en.') !== FALSE) {
-            $langF = '../_nav/langs/' . $l . '.po' . (($l == 'en') ? 't' : '');
-        }
-        if (file_exists($langF)) {
-            $stat = _po_diff($l, $resource);
-            $num_of_fuzzy_or_missing = count($stat['fuzzy_or_missing']);
-            $num_of_untranslated     = count($stat['notrans']);
-        } else { // file $langF doesn't exists
-            $stat = _po_diff('en', $resource);
-            $num_of_fuzzy_or_missing = 0;
-            $num_of_untranslated     = count($stat['notrans']);
-        }
-        // unify resource names, navigation is a special exception
-        $resource_name = str_replace(array('../_nav/langs/en', 'en/', '.pot'), array('nav', '', ''), $f);
-        // create pt_BR from pt-br and alike to unify languages
-        $web_language_code = locale_hyphen_underscore($l, true);
-        $num_of_not_fully_trans = $num_of_fuzzy_or_missing + $num_of_untranslated;
-        $report[] = array(
-            'num_of_all_strings'     => $stat['a'],
-            'resource_name'          => $resource_name,
-            'web_language_code'      => $web_language_code,
-            'num_of_not_fully_trans' => $num_of_not_fully_trans,
-        );
+if ('Webpages' == $resource_type) {
+    $enFiles = array_merge(array('../_nav/langs/en.pot'), get_lang_references('*.pot')); // added navigation file
+    $report = generating_report(get_other_langs(), $enFiles);
+} else if ('Documentation' == $resource_type) {
+    foreach ($git_resources as $git_resource_details) {
+        $git_path = $git_resource_details['git_path'];
+        $tx_name  = $git_resource_details['tx_name'];
+        $pot_name = $git_resource_details['pot_name'];
+        $raw_html_dump = file_get_contents($git_path);
+        // list all po files from links within $raw_html_dump
+        preg_match_all("/('>)([a-z_A-Z]+)(\.po<)/", $raw_html_dump, $language_codes);
+        $one_report = generating_report(array_values($language_codes[2]), array($tx_name), $git_path, $pot_name);
+        $report = array_merge($one_report, $report);
     }
 }
 
@@ -183,7 +239,7 @@ foreach ($tx_resources_info as $tx_resource_info) {
         if (!in_array($tx_resource_language, $tx_all_languages)) {
             $tx_all_languages[] = $tx_resource_language;
         }
-        if ($tx_lang_completed_treshold <= $completed) {
+        if ($tx_lang_completed_treshold <= $completed && 'en' != $tx_resource_language) {
             $tx_all_langs_above_treshold[$tx_resource_language][] = $tx_resource_name;
         }
         foreach ($report as $git_resource_info) {
@@ -204,15 +260,20 @@ foreach ($tx_resources_info as $tx_resource_info) {
 
                 $git_resource_num_of_all_strings = $git_resource_info['num_of_all_strings'];
                 $git_resource_untrans_in_lang    = $git_resource_info['num_of_not_fully_trans'];
+                $git_resource_path               = $git_resource_info['webgit_path'];
 
                 // compare l10n level
                 if ($tx_resource_untrans_in_lang != $git_resource_untrans_in_lang) {
                     $tx_git_difference[$tx_resource_language][$git_resource_name]['tx_num_of_all_strings'] = $tx_resource_all_strings_in_lang;
                     $tx_git_difference[$tx_resource_language][$git_resource_name]['tx_untranslated']       = $tx_resource_untrans_in_lang;
                     $tx_git_difference[$tx_resource_language][$git_resource_name]['git_untranslated']      = $git_resource_untrans_in_lang;
+                    $tx_git_difference[$tx_resource_language][$git_resource_name]['webgit_path']           = $git_resource_path;
                 }
                 // this bellow should normaly only happen inside of a tx sync window so remember it
-                if ($tx_resource_all_strings_in_lang != $git_resource_num_of_all_strings) {
+                if (0 == $git_resource_num_of_all_strings) {
+                    $parse_error = "It seems that parsing of $git_resource_name.po file failed in $git_resource_language language.";
+                    $errors['po_file_parse'] = $parse_error; // [$git_resource_name][$git_resource_language]
+                } else if ($tx_resource_all_strings_in_lang != $git_resource_num_of_all_strings) {
                     if (!in_array($git_resource_name, $nonequal_num_of_all_strings)) {
                         $nonequal_num_of_all_strings[] = $git_resource_name;
                     }
@@ -264,23 +325,23 @@ if (0 < count($tx_only_languages)) {
 
 // preparing error texts
 if (0 < count($git_only_languages)) {
-    $error                         = "Some languages (" . implode(", ", $git_only_languages) . ") are present in our git";
-    $errors['git_only_languages'] .= "$error but they are bellow treshold ($tx_lang_completed_treshold %) in Transifex. Please report that.";
+    $error                        = "Some languages (" . implode(", ", $git_only_languages) . ") are present in our git";
+    $errors['git_only_languages'] = "$error but they are bellow treshold ($tx_lang_completed_treshold %) in Transifex.";
 }
 
 if (0 < count($tx_only_resources)) {
-    $error                        = "Some resources (" . implode(", ", $tx_only_resources) . ") are present only in Transifex";
-    $errors['tx_only_resources'] .= "$error but not in our git. Please report that.";
+    $error                       = "Some resources (" . implode(", ", $tx_only_resources) . ") are present only in Transifex";
+    $errors['tx_only_resources'] = "$error but not in our git. Please report that.";
 }
 
 if (0 < count($git_only_resources)) {
-    $error                         = "Some resources (" . implode(", ", $git_only_resources) . ") are present only in our git";
-    $errors['git_only_resources'] .= "$error but not in Transifex. Please report that.";
+    $error                        = "Some resources (" . implode(", ", $git_only_resources) . ") are present only in our git";
+    $errors['git_only_resources'] = "$error but not in Transifex. Please report that.";
 }
 
 if (0 < count($nonequal_num_of_all_strings)) {
-    $error                                  = "Some resources (" . implode(", ", $nonequal_num_of_all_strings) . ") have different number of all strings";
-    $errors['nonequal_num_of_all_strings'] .= "$error between our git and Transifex. Please report that on the mailing list if it happens several days.";
+    $error                                 = "Some resources (" . implode(", ", $nonequal_num_of_all_strings) . ") have different number of all strings";
+    $errors['nonequal_num_of_all_strings'] = "$error between our git and Transifex. Please report that on the mailing list if it happens several days.";
 }
 
 // making a list of resources with Tx/git differences
@@ -310,7 +371,7 @@ if (0 < $num_of_resources_w_difference && $num_for_switch_table_to_list > $num_o
         $j   = 0;
         $row = array();
         $first_col_pass = TRUE;
-        if ($first_row_pass || 0 == $i % 10) {
+        if ($first_row_pass || 0 == $i % 12) {
             $array_chunks = array_chunk($all_resources_with_tx_git_diff, $col_repeat = 8, TRUE);
             foreach ($array_chunks as $array_chunk) {
                 $row = array_merge($row, array('&nbsp;'), $array_chunk);
@@ -329,6 +390,7 @@ if (0 < $num_of_resources_w_difference && $num_for_switch_table_to_list > $num_o
                     $stat_data['num_of_all'] = $one_language_array[$resource_with_tx_git_diff]['tx_num_of_all_strings'];
                     $stat_data['tx_untran']  = $one_language_array[$resource_with_tx_git_diff]['tx_untranslated'];
                     $stat_data['git_untran'] = $one_language_array[$resource_with_tx_git_diff]['git_untranslated'];
+                    $stat_data['git_path']   = $one_language_array[$resource_with_tx_git_diff]['webgit_path'];
 
                     $cell  = "<span class=\"stat\">";
                     $cell .= build_links($resource_with_tx_git_diff, $one_language, $resource_type, $stat_data);
@@ -367,6 +429,7 @@ if (0 < $num_of_resources_w_difference) {
             $stat_data['num_of_all'] = $one_language_array[$one_language_resource]['tx_num_of_all_strings'];
             $stat_data['tx_untran']  = $one_language_array[$one_language_resource]['tx_untranslated'];
             $stat_data['git_untran'] = $one_language_array[$one_language_resource]['git_untranslated'];
+            $stat_data['git_path']   = $one_language_array[$one_language_resource]['webgit_path'];
 
             $list_of_lang_diff .= ($miss_first ? '' :', ');
             $list_of_lang_diff .= "<span class=\"italic\">$one_language_resource</span> <span class=\"stat\">(";
@@ -390,6 +453,7 @@ if (0 < $num_of_resources_w_difference) {
                 $stat_data['num_of_all'] = $one_language_array[$resource_with_tx_git_diff]['tx_num_of_all_strings'];
                 $stat_data['tx_untran']  = $one_language_array[$resource_with_tx_git_diff]['tx_untranslated'];
                 $stat_data['git_untran'] = $one_language_array[$resource_with_tx_git_diff]['git_untranslated'];
+                $stat_data['git_path']   = $one_language_array[$resource_with_tx_git_diff]['webgit_path'];
 
                 $list_of_resource_diff .= ($miss_first ? '' :', ');
                 $list_of_resource_diff .= '<span class="italic">' . $langs[locale_underscore_to_hyphen($one_language)];
@@ -404,6 +468,19 @@ if (0 < $num_of_resources_w_difference) {
     $list_of_resource_diff .= '</ul>' . PHP_EOL;
 }
 
+// print links for diferent reports
+if ('Webpages' != $resource_type) {
+    $links[] = '<a href="./report_tx_git.php?c=Webpages">website translations</a>';
+}
+if ('Documentation' != $resource_type) {
+    $links[] = '<a href="./report_tx_git.php?c=Documentation">documentation translations</a>';
+}
+if ('Cauldron' != $resource_type) {
+    $links[] = '<a href="./report_tx_git.php?c=Cauldron">software translations</a>';
+}
+$reports_links = '<p>You can also see reports about differences in ' . implode(" or ", $links) . '.</p>';
+echo $reports_links . PHP_EOL;
+
 // print out any errors
 if (0 < count($errors)) {
     $error_text = '<ul>';