first steps as a preparation of l10n report (only diff.php for now - 3 exceptions) to gettext system

4 files changed, 131 insertions, 23 deletions

diff --git a/langs.inc.php b/langs.inc.php
index a30e44a24..5a358587e 100644
--- a/langs.inc.php
+++ b/langs.inc.php
@@ -2,6 +2,8 @@
 /**
 */
 
+$g_app_root = realpath(dirname(__FILE__));
+define('G_APP_ROOT', $g_app_root);
 define('G_VHOST', $_SERVER['SERVER_NAME']);
 
 // languages for home
@@ -449,7 +451,12 @@ function _lang_load($locale, $domain) { return i18n::_lang_load($locale, $domain
 */
 function read_translation_file($locale, $name_of_translation)
 {
-    return phpmo_parse_po_file(G_APP_ROOT . '/langs/' . $locale . '/' . $name_of_translation .'.po');
+    if($locale == 'en') {
+        $filename = $name_of_translation . '.pot';
+    } else {
+        $filename = $name_of_translation . '.po';
+    }
+    return phpmo_parse_po_file(G_APP_ROOT . '/langs/' . $locale . '/' . $filename);
 }
 
 /**
diff --git a/langs.php b/langs.php
index 0c858137b..ddf3bf1a3 100644
--- a/langs.php
+++ b/langs.php
@@ -20,8 +20,8 @@ if ($_SERVER['HTTP_HOST'] == 'www-test.mageia.org') {
     ini_set('log_errors', true);
 }
 
-$g_app_root = realpath(dirname(__FILE__));
-define('G_APP_ROOT', $g_app_root);
+//$g_app_root = realpath(dirname(__FILE__));
+//define('G_APP_ROOT', $g_app_root);
 
 $g_donate_amount = '--';
 $g_amount_remain = 'EUR 10,244.46';
diff --git a/langs/diff.php b/langs/diff.php
index c231b9131..bc6e0ab5a 100644
--- a/langs/diff.php
+++ b/langs/diff.php
@@ -6,8 +6,6 @@
 
 include 'lib.php';
 
-define('APP_ROOT', realpath(__DIR__ . '/..'));
-
 $source_file = isset($_GET['s']) ? strip_tags(trim($_GET['s'])) : null;
 $target_lang = isset($_GET['l']) ? strip_tags(trim($_GET['l'])) : null;
 
@@ -21,43 +19,55 @@ if (!file_exists($source_file)) {
     die('no source');
 }
 
-$target_file = _lang_file_switch($source_file, $target_lang);
+if (strrpos($source_file, '.pot')) {
+    $gettext = true;
+    $resource = _extract_resource($source_file);
+    $target_file = _po_file_switch($source_file, $target_lang);
+} else {
+    $gettext = false;
+    $resource = _extract_resource($source_file, '.en.lang');
+    $target_file = _lang_file_switch($source_file, $target_lang);
+}
 
 if (!file_exists($target_file)) {
     die('no target');
 }
 
-if($source_file =='en/about/constitution.en.lang') {
+if($resource =='about/constitution') {
     $constitution = true;
 } else {
     $constitution = false;
 }
 
-if($source_file =='en/about/license.en.lang') {
+if($resource =='about/license') {
     $license = true;
 } else {
     $license = false;
 }
 
 if ($source_file == '../_nav/langs/en.lang') {
-    $file   = substr(_lang_file_switch($source_file, $target_lang), 14);
+    $file   = substr($target_file, 14);
     $note   = '<span style="color: red; font-weight: bold"> NOTE: THIS IS TRANSLATION OF NAVIGATION SO LOCATION FOR COMMIT IS DIFFERENT!</span>';
     $commit = '<a href="//svnweb.mageia.org/web/www/trunk/_nav/langs/' . $file . '?view=markup">web/www/trunk/_nav/langs/' . $file . '</a> and for now to the <a href="//svnweb.mageia.org/web/nav/langs/' . $file . '?view=markup">web/nav/langs/' . $file . '</a> too';
 } else {
-    $file   = _lang_file_switch($source_file, $target_lang);
+    $file   = $target_file;
     $note   = '';
     $commit = '<a href="//svnweb.mageia.org/web/www/trunk/langs/' . $file . '?view=markup">web/www/trunk/langs/' . $file . '</a>';
 }
 
-$source_file = realpath($source_file);
-$target_file = realpath($target_file);
+$source_file_path = realpath($source_file);
+$target_file_path = realpath($target_file);
 
-if (false === strstr($source_file, '/langs/') ||
-    false == strstr($target_file, '/langs/')) {
+if (false === strstr($source_file_path, '/langs/') ||
+    false == strstr($target_file_path, '/langs/')) {
     die('no sorry');
 }
 
-$diff = _lang_diff($source_file, $target_file);
+if ($gettext) {
+    $diff = _po_diff($target_lang, $resource);
+} else {
+    $diff = _lang_diff($source_file_path, $target_file_path);
+}
 
 $issues = array(
     'missing' => 'missing strings',
@@ -68,15 +78,15 @@ $issues = array(
 
 $s = '<a href="report.php?l=' . $target_lang . '">&laquo; back to the report page for ' . $langs[$target_lang] . '</a>';
 $s .= sprintf('<h1>Differences between %s source and %s target</h1>',
-    substr($source_file, strpos($source_file, '/langs/')+7), $target_lang);
+    $source_file, $target_lang);
 
 if($constitution) {
     $s .= '<h2 style="color: red;">Please translate constitution fully into ' . $langs[$target_lang] . ' first</h2>';
     $s .= '<p>You can find it in <a href="//svnweb.mageia.org/org/constitution/">svn</a>. ';
 
-    $constitution_results = aproximate_number_of_untranslated_constitution_lines(APP_ROOT, 'en');
+    $constitution_results = aproximate_number_of_untranslated_constitution_lines(G_APP_ROOT, 'en');
     $unique_lines_in_eng_constitution = $constitution_results['unique_lines_in_constitution'];
-    $lang_constitution_results = aproximate_number_of_untranslated_constitution_lines(APP_ROOT, $target_lang, $unique_lines_in_eng_constitution);
+    $lang_constitution_results = aproximate_number_of_untranslated_constitution_lines(G_APP_ROOT, $target_lang, $unique_lines_in_eng_constitution);
     $constitution_readable = $lang_constitution_results['constitution_readable'];
     $diff['untranslated_lines_in_constitution'] = $lang_constitution_results['untranslated_lines_in_constitution']; // add untranslated license sentences
     $dest_constitution = sprintf('%s/%s_%s.md', 'href="//svnweb.mageia.org/web/www/trunk/en/about/constitution', 'mageia.org_statutes', ($constitution_readable ? $target_lang : 'en'));
@@ -129,7 +139,7 @@ header('Content-Type: text/html;charset=utf-8');
     <meta charset="utf-8">
     <meta name="robot" content="noindex,nofollow,nosnippet">
     <?php echo sprintf('<title>Differences between %s source and %s target</title>',
-    substr($source_file, strpos($source_file, '/langs/')+7), $target_lang); ?>
+    $source_file, $target_lang); ?>
     <style>
     pre { background: #eee; padding: 0.6em; }
     </style>
diff --git a/langs/lib.php b/langs/lib.php
index f7422167f..05740418d 100644
--- a/langs/lib.php
+++ b/langs/lib.php
@@ -21,13 +21,12 @@ include '../langs.inc.php';
  * - missing strings (in $a, not in $b)
  * - extra strings (in $b, not in $a)
  * - untranslated strings (same in $a and $b, or empty in $b)
+ * - duplicate strings in $b
  *
  * @param string $a file name
  * @param string $b file name
  *
  * @return array
- *
- * @todo some strings may be left untranslated on purpose
 */
 function _lang_diff($a, $b)
 {
@@ -70,7 +69,60 @@ function _lang_diff($a, $b)
     );
 }
 
-function _lang_diff_stats($a, $b)
+/**
+ * Diff pot and po files, to get:
+ * - source (pot) strings count
+ * - missing strings in target
+ * - untranslated strings in target
+ * - empty array for extra and duplicate strings for backward compatibility
+ *
+ * @param string $locale locale name ('sl')
+ * @param string $resource file name ('about/license')
+ *
+ * @return array
+*/
+function _po_diff($locale, $resource)
+{
+    $source_l = read_translation_file('en', $resource);
+    $target_l = read_translation_file($locale, $resource);
+
+    $pot_strings = array();
+    $untrans = array();
+    $missing = array();
+
+    foreach ($source_l as $escaped_string => $subarray) {
+        if ($escaped_string == $subarray["msgid"]) { // filter out header
+            $pot_strings[$escaped_string] = $subarray["msgid"];
+        }
+    }
+
+    foreach ($target_l as $escaped_string => $subarray) {
+        if ($escaped_string == $subarray["msgid"]) { // filter out header
+            $po_strings[$escaped_string] = $subarray["msgstr"][0];
+        }
+    }
+
+    foreach ($pot_strings as $escaped_string => $translated_string) {
+        if (isset($po_strings[$escaped_string])) {
+            if (empty($po_strings[$escaped_string])) {
+                $untrans[] = $escaped_string;
+            }
+        } else {
+            $missing[] = $escaped_string;
+        }
+    }
+
+    return array(
+        'a'       => count($pot_strings), // # of original strings
+//        'b'       => count($po_strings), // # of target strings
+        'missing' => $missing, // probably could be empty array() of missing strings
+        'notrans' => $untrans, // array of untranslated strings
+        'extra'   => array(), // could be empty array() of unused (old) strings
+        'dup_str' => array(), // could be empty array() of duplicate strings
+    );
+}
+
+/*function _lang_diff_stats($a, $b)
 {
     $diff = _lang_diff($a, $b);
 
@@ -81,7 +133,7 @@ function _lang_diff_stats($a, $b)
     $diff['correct'] = $diff['b'] - $diff['notrans'] - $diff['missing'];
 
     return $diff;
-}
+} /**/
 
 if ( ! function_exists('glob_recursive'))
 {
@@ -111,12 +163,51 @@ if ( ! function_exists('glob_recursive'))
     }
 }
 
+/**
+ * Create 'sl/about/license.sl.lang'
+ * from 'en/about/license.en.lang'
+ *
+ * @param string $s file name with path
+ * @param string $l locale name
+ *
+ * @return string
+*/
 function _lang_file_switch($s, $l)
 {
     $s = str_replace('en.lang', $l . '.lang', $s);
     return str_replace('en/', $l . '/', $s);
 }
 
+/**
+ * Create 'sl/about/license.po'
+ * from 'en/about/license.pot'
+ *
+ * @param string $s file name with path
+ * @param string $l locale name
+ *
+ * @return string
+*/
+function _po_file_switch($s, $l)
+{
+    $s = str_replace('.pot', '.po', $s);
+    return str_replace('en/', $l . '/', $s);
+}
+
+/**
+ * Create 'about/license'
+ * from 'en/about/license.pot' or 'en/about/license.en.lang'
+ *
+ * @param string $source_file file name with path
+ * @param string $extension file extension to remove
+ *
+ * @return string
+*/
+function _extract_resource($source_file, $extension = '.pot')
+{
+    $resource = str_replace($extension, '', $source_file);
+    return str_replace('en/', '', $resource);
+}
+
 function get_lang_references()
 {
     return glob_recursive('en/*', GLOB_MARK);